From b2748c6e99239ff6803ba0da76c362790c8be192 Mon Sep 17 00:00:00 2001
From: Lucas Franceschino <lucas.franceschino@inria.fr>
Date: Mon, 25 May 2020 19:07:38 +0200
Subject: [PATCH 001/882] Make `functionArgs` primitive accept primops

---
 src/libexpr/primops.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 0a4236da4..5875457ac 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1346,6 +1346,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
+    if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) {
+        state.mkAttrs(v, 0);
+        return;
+    }
     if (args[0]->type != tLambda)
         throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
 

From 7130f0a3a6455cf27863f0bbccf0974cfcc82fa2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 Jun 2020 04:04:41 +0000
Subject: [PATCH 002/882] Don't need abstract `struct Derivation` in
 local-store

---
 src/libstore/local-store.hh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index e17cc45ae..872500957 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -23,9 +23,6 @@ namespace nix {
 const int nixSchemaVersion = 10;
 
 
-struct Derivation;
-
-
 struct OptimiseStats
 {
     unsigned long filesLinked = 0;

From 18493fd9c48676ab26854739db67ac5d76ff9347 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 Jun 2020 03:56:48 +0000
Subject: [PATCH 003/882] Move some Store functions from derivations.cc to
 store-api.cc

This further continues with the dependency inverstion. Also I just went
ahead and exposed `parseDerivation`: it seems like the more proper
building block, and not a bad thing to expose if we are trying to be
less wedded to drv files on disk anywas.
---
 src/libexpr/primops.cc      |  2 +-
 src/libstore/derivations.cc | 31 +------------------------------
 src/libstore/derivations.hh |  2 +-
 src/libstore/store-api.cc   | 20 +++++++++++++++++++-
 src/nix/repl.cc             | 28 ++++++++++++++++------------
 5 files changed, 38 insertions(+), 45 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 907f15246..0c2371c43 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -103,7 +103,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 
     // FIXME
     if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
-        Derivation drv = readDerivation(*state.store, realPath);
+        Derivation drv = state.store->readDerivation(state.store->parseStorePath(path));
         Value & w = *state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
         Value * v2 = state.allocAttr(w, state.sDrvPath);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index b95f7bfdc..ea30813fa 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -3,7 +3,6 @@
 #include "globals.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
-#include "fs-accessor.hh"
 #include "istringstream_nocopy.hh"
 
 namespace nix {
@@ -120,7 +119,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static Derivation parseDerivation(const Store & store, const string & s)
+Derivation parseDerivation(const Store & store, const string & s)
 {
     Derivation drv;
     istringstream_nocopy str(s);
@@ -173,34 +172,6 @@ static Derivation parseDerivation(const Store & store, const string & s)
 }
 
 
-Derivation readDerivation(const Store & store, const Path & drvPath)
-{
-    try {
-        return parseDerivation(store, readFile(drvPath));
-    } catch (FormatError & e) {
-        throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
-    }
-}
-
-
-Derivation Store::derivationFromPath(const StorePath & drvPath)
-{
-    ensurePath(drvPath);
-    return readDerivation(drvPath);
-}
-
-
-Derivation Store::readDerivation(const StorePath & drvPath)
-{
-    auto accessor = getFSAccessor();
-    try {
-        return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
-    } catch (FormatError & e) {
-        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
-    }
-}
-
-
 static void printString(string & res, std::string_view s)
 {
     char buf[s.size() * 2 + 2];
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 856aae59f..2adbf6b94 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -77,7 +77,7 @@ StorePath writeDerivation(ref<Store> store,
     const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
-Derivation readDerivation(const Store & store, const Path & drvPath);
+Derivation parseDerivation(const Store & store, const string & s);
 
 // FIXME: remove
 bool isDerivation(const string & fileName);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index aae227bae..0147445ff 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,11 +1,11 @@
 #include "crypto.hh"
+#include "fs-accessor.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
 #include "json.hh"
-#include "derivations.hh"
 #include "url.hh"
 
 #include <future>
@@ -821,6 +821,24 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
 }
 
 
+Derivation Store::derivationFromPath(const StorePath & drvPath)
+{
+    ensurePath(drvPath);
+    return readDerivation(drvPath);
+}
+
+
+Derivation Store::readDerivation(const StorePath & drvPath)
+{
+    auto accessor = getFSAccessor();
+    try {
+        return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
+    } catch (FormatError & e) {
+        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
+    }
+}
+
+
 }
 
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 4bcaaeebf..b46f7015f 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -66,7 +66,7 @@ struct NixRepl : gc
     void mainLoop(const std::vector<std::string> & files);
     StringSet completePrefix(string prefix);
     bool getLine(string & input, const std::string &prompt);
-    Path getDerivationPath(Value & v);
+    StorePath getDerivationPath(Value & v);
     bool processLine(string line);
     void loadFile(const Path & path);
     void initEnv();
@@ -377,13 +377,16 @@ bool isVarName(const string & s)
 }
 
 
-Path NixRepl::getDerivationPath(Value & v) {
+StorePath NixRepl::getDerivationPath(Value & v) {
     auto drvInfo = getDerivation(*state, v, false);
     if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    Path drvPath = drvInfo->queryDrvPath();
-    if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath)))
-        throw Error("expression did not evaluate to a valid derivation");
+    Path drvPathRaw = drvInfo->queryDrvPath();
+    if (drvPathRaw == "")
+        throw Error("expression did not evaluate to a valid derivation (no drv path)");
+    StorePath drvPath = state->store->parseStorePath(drvPathRaw);
+    if (!state->store->isValidPath(drvPath))
+        throw Error("expression did not evaluate to a valid derivation (invalid drv path)");
     return drvPath;
 }
 
@@ -476,29 +479,30 @@ bool NixRepl::processLine(string line)
         evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
         state->callFunction(f, v, result, Pos());
 
-        Path drvPath = getDerivationPath(result);
-        runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
+        StorePath drvPath = getDerivationPath(result);
+        runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
         Value v;
         evalString(arg, v);
-        Path drvPath = getDerivationPath(v);
+        StorePath drvPath = getDerivationPath(v);
+        Path drvPathRaw = state->store->printStorePath(drvPath);
 
         if (command == ":b") {
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                auto drv = readDerivation(*state->store, drvPath);
+            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
+                auto drv = state->store->readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)
                     std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.path));
             }
         } else if (command == ":i") {
-            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
+            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});
         } else {
-            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
+            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw});
         }
     }
 

From 88cf6ffce3c01f4f1c50250ef46c0d7bf23f41c7 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 16:27:00 -0400
Subject: [PATCH 004/882] Rename logging->stdout to logging->stdout_

musl doesn't like this identifier
---
 src/libutil/logging.hh  | 2 +-
 src/nix/add-to-store.cc | 2 +-
 src/nix/eval.cc         | 2 +-
 src/nix/hash.cc         | 4 ++--
 src/nix/ls.cc           | 4 ++--
 src/nix/show-config.cc  | 2 +-
 src/nix/why-depends.cc  | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index b1583eced..46deb89f7 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -87,7 +87,7 @@ public:
     virtual void writeToStdout(std::string_view s);
 
     template<typename... Args>
-    inline void stdout(const std::string & fs, const Args & ... args)
+    inline void stdout_(const std::string & fs, const Args & ... args)
     {
         boost::format f(fs);
         formatHelper(f, args...);
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f9d6de16e..745c24748 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -58,7 +58,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             store->addToStore(info, source);
         }
 
-        logger->stdout("%s", store->printStorePath(info.path));
+        logger->stdout_("%s", store->printStorePath(info.path));
     }
 };
 
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 26e98ac2a..53ec8c920 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -65,7 +65,7 @@ struct CmdEval : MixJSON, InstallableCommand
             printValueAsJSON(*state, true, *v, jsonOut, context);
         } else {
             state->forceValueDeep(*v);
-            logger->stdout("%s", *v);
+            logger->stdout_("%s", *v);
         }
     }
 };
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index b97c6d21f..cdc8bf767 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -69,7 +69,7 @@ struct CmdHash : Command
 
             Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
-            logger->stdout(h.to_string(base, base == SRI));
+            logger->stdout_(h.to_string(base, base == SRI));
         }
     }
 };
@@ -103,7 +103,7 @@ struct CmdToBase : Command
     void run() override
     {
         for (auto s : args)
-            logger->stdout(Hash(s, ht).to_string(base, base == SRI));
+            logger->stdout_(Hash(s, ht).to_string(base, base == SRI));
     }
 };
 
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index d2157f2d4..59922a8de 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -37,11 +37,11 @@ struct MixLs : virtual Args, MixJSON
                 auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
                 if (st.type == FSAccessor::Type::tSymlink)
                     line += " -> " + accessor->readLink(curPath);
-                logger->stdout(line);
+                logger->stdout_(line);
                 if (recursive && st.type == FSAccessor::Type::tDirectory)
                     doPath(st, curPath, relPath, false);
             } else {
-                logger->stdout(relPath);
+                logger->stdout_(relPath);
                 if (recursive) {
                     auto st = accessor->stat(curPath);
                     if (st.type == FSAccessor::Type::tDirectory)
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 4fd8886de..a97dc42f9 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -25,7 +25,7 @@ struct CmdShowConfig : Command, MixJSON
             std::map<std::string, Config::SettingInfo> settings;
             globalConfig.getSettings(settings);
             for (auto & s : settings)
-                logger->stdout("%s = %s", s.first, s.second.value);
+                logger->stdout_("%s = %s", s.first, s.second.value);
         }
     }
 };
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 167c974ee..5e4d5fdcf 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -152,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand
             auto pathS = store->printStorePath(node.path);
 
             assert(node.dist != inf);
-            logger->stdout("%s%s%s%s" ANSI_NORMAL,
+            logger->stdout_("%s%s%s%s" ANSI_NORMAL,
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
                 firstPad != "" ? "→ " : "",

From 07dae2ff7727b915a1b687cdec9aca894d7c2f72 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 17:19:11 -0400
Subject: [PATCH 005/882] Setup static building of nix

---
 release-common.nix |  37 ++++++++------
 release.nix        | 117 ++++++++++++++++++++++++---------------------
 2 files changed, 85 insertions(+), 69 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index 4316c3c23..2cf9c233e 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -1,4 +1,4 @@
-{ pkgs }:
+{ pkgs, enableStatic }:
 
 with pkgs;
 
@@ -30,35 +30,42 @@ rec {
   });
 
   configureFlags =
-    lib.optionals stdenv.isLinux [
+    lib.optionals (!enableStatic && stdenv.isLinux) [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
+  nativeBuildDeps =
+    [
+      buildPackages.bison
+      buildPackages.flex
+      buildPackages.libxml2
+      buildPackages.libxslt
+      buildPackages.docbook5
+      buildPackages.docbook_xsl_ns
+      buildPackages.autoreconfHook
+      buildPackages.pkgconfig
+
+      # Tests
+      buildPackages.git
+      buildPackages.mercurial
+      buildPackages.ipfs
+    ];
+
   buildDeps =
-    [ bison
-      flex
-      libxml2
-      libxslt
-      docbook5
-      docbook_xsl_ns
+    [ autoreconfHook
       autoconf-archive
-      autoreconfHook
 
       curl
       bzip2 xz brotli zlib editline
-      openssl pkgconfig sqlite
+      openssl sqlite
       libarchive
       boost
       nlohmann_json
-
-      # Tests
-      git
-      mercurial
       gmock
     ]
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
+    ++ lib.optional (!enableStatic && (stdenv.isLinux || stdenv.isDarwin))
       ((aws-sdk-cpp.override {
         apis = ["s3" "transfer"];
         customMemoryManagement = false;
diff --git a/release.nix b/release.nix
index fbf9e4721..d2785be13 100644
--- a/release.nix
+++ b/release.nix
@@ -12,63 +12,72 @@ let
     builtins.readFile ./.version
     + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
 
+  buildFun = pkgs: enableStatic:
+  with pkgs; with import ./release-common.nix { inherit pkgs enableStatic; };
+  stdenv.mkDerivation {
+    name = "nix-${version}";
+
+    src = nix;
+
+    outputs = [ "out" "dev" "doc" ];
+
+    buildInputs = buildDeps;
+
+    nativeBuildInputs = nativeBuildDeps;
+
+    propagatedBuildInputs = propagatedDeps;
+
+    preConfigure =
+      lib.optionalString (!enableStatic) ''
+        # Copy libboost_context so we don't get all of Boost in our closure.
+        # https://github.com/NixOS/nixpkgs/issues/45462
+        mkdir -p $out/lib
+        cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+        rm -f $out/lib/*.a
+        ${lib.optionalString stdenv.isLinux ''
+          chmod u+w $out/lib/*.so.*
+          patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+        ''}
+
+        (cd perl; autoreconf --install --force --verbose)
+      '';
+
+    configureFlags = configureFlags ++
+      [ "--sysconfdir=/etc" ];
+
+    dontUpdateAutotoolsGnuConfigScripts = true;
+
+    enableParallelBuilding = true;
+
+    makeFlags = [ "profiledir=$(out)/etc/profile.d" "PRECOMPILE_HEADERS=0" ];
+
+    installFlags = "sysconfdir=$(out)/etc";
+
+    postInstall = ''
+      mkdir -p $doc/nix-support
+      echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+    '';
+
+    doCheck = true;
+
+    doInstallCheck = true;
+    installCheckFlags = "sysconfdir=$(out)/etc";
+
+    separateDebugInfo = !enableStatic;
+
+    stripAllList = ["bin"];
+  };
+
+
   jobs = rec {
 
+
+    build-static = pkgs.lib.genAttrs systems (system:
+      buildFun (import nixpkgs { inherit system; }).pkgsStatic true);
+
+
     build = pkgs.lib.genAttrs systems (system:
-
-      let pkgs = import nixpkgs { inherit system; }; in
-
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      stdenv.mkDerivation {
-        name = "nix-${version}";
-
-        src = nix;
-
-        outputs = [ "out" "dev" "doc" ];
-
-        buildInputs = buildDeps;
-
-        propagatedBuildInputs = propagatedDeps;
-
-        preConfigure =
-          ''
-            # Copy libboost_context so we don't get all of Boost in our closure.
-            # https://github.com/NixOS/nixpkgs/issues/45462
-            mkdir -p $out/lib
-            cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-            rm -f $out/lib/*.a
-            ${lib.optionalString stdenv.isLinux ''
-              chmod u+w $out/lib/*.so.*
-              patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-            ''}
-
-            (cd perl; autoreconf --install --force --verbose)
-          '';
-
-        configureFlags = configureFlags ++
-          [ "--sysconfdir=/etc" ];
-
-        enableParallelBuilding = true;
-
-        makeFlags = "profiledir=$(out)/etc/profile.d";
-
-        installFlags = "sysconfdir=$(out)/etc";
-
-        postInstall = ''
-          mkdir -p $doc/nix-support
-          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-        '';
-
-        doCheck = true;
-
-        doInstallCheck = true;
-        installCheckFlags = "sysconfdir=$(out)/etc";
-
-        separateDebugInfo = true;
-      });
+      buildFun (import nixpkgs { inherit system; }) false);
 
 
     perlBindings = pkgs.lib.genAttrs systems (system:

From 70719a9dd8c3d91e1d6a83d4ec9a48023cddaecf Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 17:20:06 -0400
Subject: [PATCH 006/882] Add -lz to end of linking

this is needed for static linking to work properly
---
 mk/programs.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mk/programs.mk b/mk/programs.mk
index 3fa9685c3..a96ff56af 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -32,7 +32,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) -lz
 
   $(1)_INSTALL_DIR ?= $$(bindir)
 

From da77331cb740ad7d5f39dcf6d64025610ec40555 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 17:20:29 -0400
Subject: [PATCH 007/882] Remove lazy lookup in getHome

this seems to break in Musl/Static with:

terminate called after throwing an instance of 'std::bad_function_call'
  what():  bad_function_call
---
 src/libutil/lazy.hh | 48 ---------------------------------------------
 src/libutil/util.cc |  6 +++---
 2 files changed, 3 insertions(+), 51 deletions(-)
 delete mode 100644 src/libutil/lazy.hh

diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh
deleted file mode 100644
index d073e486c..000000000
--- a/src/libutil/lazy.hh
+++ /dev/null
@@ -1,48 +0,0 @@
-#include <exception>
-#include <functional>
-#include <mutex>
-
-namespace nix {
-
-/* A helper class for lazily-initialized variables.
-
-     Lazy<T> var([]() { return value; });
-
-   declares a variable of type T that is initialized to 'value' (in a
-   thread-safe way) on first use, that is, when var() is first
-   called. If the initialiser code throws an exception, then all
-   subsequent calls to var() will rethrow that exception. */
-template<typename T>
-class Lazy
-{
-
-    typedef std::function<T()> Init;
-
-    Init init;
-
-    std::once_flag done;
-
-    T value;
-
-    std::exception_ptr ex;
-
-public:
-
-    Lazy(Init init) : init(init)
-    { }
-
-    const T & operator () ()
-    {
-        std::call_once(done, [&]() {
-            try {
-                value = init();
-            } catch (...) {
-                ex = std::current_exception();
-            }
-        });
-        if (ex) std::rethrow_exception(ex);
-        return value;
-    }
-};
-
-}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1268b146a..ebb1383f3 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,4 +1,3 @@
-#include "lazy.hh"
 #include "util.hh"
 #include "affinity.hh"
 #include "sync.hh"
@@ -511,7 +510,8 @@ std::string getUserName()
 }
 
 
-static Lazy<Path> getHome2([]() {
+static Path getHome2()
+{
     auto homeDir = getEnv("HOME");
     if (!homeDir) {
         std::vector<char> buf(16384);
@@ -523,7 +523,7 @@ static Lazy<Path> getHome2([]() {
         homeDir = pw->pw_dir;
     }
     return *homeDir;
-});
+};
 
 Path getHome() { return getHome2(); }
 

From 289558dffbccb47191e79629e955009b10b9888e Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 17:39:35 -0400
Subject: [PATCH 008/882] Add unordered_set to globals.cc header

---
 src/libstore/globals.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index bee94cbd8..fa8799314 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -8,7 +8,7 @@
 #include <thread>
 #include <dlfcn.h>
 #include <sys/utsname.h>
-
+#include <unordered_set>
 
 namespace nix {
 

From 78fadaf863536d23d45b2c480139a9f77d579f9e Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 17:44:57 -0400
Subject: [PATCH 009/882] fix release.nix eval

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index d2785be13..863db900b 100644
--- a/release.nix
+++ b/release.nix
@@ -191,7 +191,7 @@ let
     coverage =
       with pkgs;
 
-      with import ./release-common.nix { inherit pkgs; };
+      with import ./release-common.nix { inherit pkgs; enableStatic = false; };
 
       releaseTools.coverageAnalysis {
         name = "nix-coverage-${version}";

From ded65899538f6a4628e711abbbdf27ea47772742 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <matthew.bauer@obsidian.systems>
Date: Thu, 25 Jun 2020 18:04:16 -0400
Subject: [PATCH 010/882] Fixup coverage build

---
 release.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release.nix b/release.nix
index 863db900b..60cbca17b 100644
--- a/release.nix
+++ b/release.nix
@@ -200,6 +200,7 @@ let
 
         enableParallelBuilding = true;
 
+        nativeBuildInputs = nativeBuildDeps;
         buildInputs = buildDeps ++ propagatedDeps;
 
         dontInstall = false;

From 24da034bc3ae8514ae19dadfecf6038452a5290a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 29 Jun 2020 21:21:27 +0000
Subject: [PATCH 011/882] Add possibly missing `<string>` include

---
 src/libutil/types.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 3af485fa0..2170e4c93 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -4,6 +4,7 @@
 
 #include <list>
 #include <set>
+#include <string>
 #include <map>
 #include <vector>
 

From 696bb134c1c5882cf258e3c8a480b40239cb1a9a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 29 Jun 2020 21:36:09 +0000
Subject: [PATCH 012/882] Fix shell.nix

---
 shell.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shell.nix b/shell.nix
index 17aaa05ed..1addc06be 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,8 +1,8 @@
-{ useClang ? false }:
+{ useClang ? false, enableStatic ? false }:
 
 with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
 
-with import ./release-common.nix { inherit pkgs; };
+with import ./release-common.nix { inherit pkgs enableStatic; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";

From baaab2aab58aa3c47517d4ba9121a29a7ad73078 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 30 Jun 2020 14:53:40 +0000
Subject: [PATCH 013/882] Add `nativeBuildInputs` to shell.nix

---
 shell.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shell.nix b/shell.nix
index 1addc06be..75bb6ac1b 100644
--- a/shell.nix
+++ b/shell.nix
@@ -7,6 +7,8 @@ with import ./release-common.nix { inherit pkgs enableStatic; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
+  nativeBuildInputs = nativeBuildDeps;
+
   buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
 
   inherit configureFlags;

From f1c7746eb407258d77b2f7fffec0e5d4facf516a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 5 Jul 2020 21:39:04 +0000
Subject: [PATCH 014/882] See if setting -std=c++17 for perl bindings helps

---
 perl/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/Makefile b/perl/Makefile
index 7ddb0cf69..259ed7dc3 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall
+GLOBAL_CXXFLAGS += -g -Wall -std=c++17
 
 -include Makefile.config
 

From e3b394b6e8420795c1976195184921df2c2b3e83 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Thu, 16 Jul 2020 09:36:02 -0400
Subject: [PATCH 015/882] Small namespace fix

---
 src/libstore/derivations.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index d32793b6a..d170d7223 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -76,7 +76,7 @@ StorePath writeDerivation(ref<Store> store,
     const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
-Derivation parseDerivation(const Store & store, string && s);
+Derivation parseDerivation(const Store & store, std::string && s);
 
 // FIXME: remove
 bool isDerivation(const string & fileName);

From 0aa79dcc6f4e67891cdd9500ae4ce85e4189a951 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Fri, 17 Jul 2020 17:24:02 -0400
Subject: [PATCH 016/882] Remove StoreType abstraction and delegate regStore

to each Store implementation. The generic regStore implementation will
only be for the ambiguous shorthands, like "" and "auto".

This also could get us close to simplifying the daemon command.
---
 src/libstore/local-store.cc  | 16 ++++++++++++
 src/libstore/remote-store.cc | 10 +++++---
 src/libstore/store-api.cc    | 47 ++++++++++--------------------------
 src/libstore/store-api.hh    | 10 --------
 src/nix-daemon/nix-daemon.cc |  5 +++-
 src/nix/doctor.cc            |  4 +--
 6 files changed, 41 insertions(+), 51 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d49d00d6d..9752f3c5f 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1580,4 +1580,20 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
 }
 
 
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    Store::Params params2 = params;
+    if (uri == "local") {
+    } else if (hasPrefix(uri, "/")) {
+        params2["root"] = uri;
+    } else if (hasPrefix(uri, "./")) {
+        params2["root"] = absPath(uri);
+    } else {
+        return nullptr;
+    }
+    return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
+});
+
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 9af4364b7..a9fbf9f82 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -843,14 +843,18 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-static std::string uriScheme = "unix://";
+static std::string_view uriScheme = "unix://";
 
 static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
+    if (hasPrefix(uri, uriScheme))
+        return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
+    else if (uri == "daemon")
+        return std::make_shared<UDSRemoteStore>(params);
+    else
+        return nullptr;
 });
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index d37e970df..7a380f127 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -915,44 +915,23 @@ ref<Store> openStore(const std::string & uri_,
 }
 
 
-StoreType getStoreType(const std::string & uri, const std::string & stateDir)
-{
-    if (uri == "daemon") {
-        return tDaemon;
-    } else if (uri == "local" || hasPrefix(uri, "/") || hasPrefix(uri, "./")) {
-        return tLocal;
-    } else if (uri == "" || uri == "auto") {
-        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
-            return tLocal;
-        else if (pathExists(settings.nixDaemonSocketFile))
-            return tDaemon;
-        else
-            return tLocal;
-    } else {
-        return tOther;
-    }
-}
-
-
+// Specific prefixes are handled by the specific types of store, while here we
+// handle the general cases not covered by the other ones.
 static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
-    switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
-        case tDaemon:
-            return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
-        case tLocal: {
-            Store::Params params2 = params;
-            if (hasPrefix(uri, "/")) {
-                params2["root"] = uri;
-            } else if (hasPrefix(uri, "./")) {
-                params2["root"] = absPath(uri);
-            }
-            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
-        }
-        default:
-            return nullptr;
-    }
+     auto stateDir = get(params, "state").value_or(settings.nixStateDir);
+     if (uri == "" || uri == "auto") {
+        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
+            return std::make_shared<LocalStore>(params);
+        else if (pathExists(settings.nixDaemonSocketFile))
+            return std::make_shared<UDSRemoteStore>(params);
+        else
+            return std::make_shared<LocalStore>(params);
+     } else {
+         return nullptr;
+     }
 });
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a4be0411e..c38290add 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -796,16 +796,6 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
     const Store::Params & extraParams = Store::Params());
 
 
-enum StoreType {
-    tDaemon,
-    tLocal,
-    tOther
-};
-
-
-StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
-    const std::string & stateDir = settings.nixStateDir);
-
 /* Return the default substituter stores, defined by the
    ‘substituters’ option and various legacy options. */
 std::list<ref<Store>> getDefaultSubstituters();
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index bcb86cbce..b52cd7989 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -1,5 +1,6 @@
 #include "shared.hh"
 #include "local-store.hh"
+#include "remote-store.hh"
 #include "util.hh"
 #include "serialise.hh"
 #include "archive.hh"
@@ -277,7 +278,9 @@ static int _main(int argc, char * * argv)
         initPlugins();
 
         if (stdio) {
-            if (getStoreType() == tDaemon) {
+            if (openUncachedStore().dynamic_pointer_cast<UDSRemoteStore>()) {
+                // FIXME Use the connection the UDSRemoteStore opened
+
                 //  Forward on this connection to the real daemon
                 auto socketPath = settings.nixDaemonSocketFile;
                 auto s = socket(PF_UNIX, SOCK_STREAM, 0);
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 82e92cdd0..683e91446 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -49,9 +49,7 @@ struct CmdDoctor : StoreCommand
     {
         logger->log("Running checks against store uri: " + store->getUri());
 
-        auto type = getStoreType();
-
-        if (type < tOther) {
+        if (store.dynamic_pointer_cast<LocalFSStore>()) {
             success &= checkNixInPath();
             success &= checkProfileRoots(store);
         }

From 650ae14ceda72dcb294bde4d08988a7ed26ab0ff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 13:51:11 +0200
Subject: [PATCH 017/882] Markdown test

---
 doc/manual/command-ref/nix-copy-closure.md |  78 +++++++++
 doc/manual/introduction/about-nix.md       | 181 +++++++++++++++++++++
 doc/manual/introduction/quick-start.md     |  79 +++++++++
 doc/manual/manual.md                       |   8 +
 flake.nix                                  |  21 +++
 5 files changed, 367 insertions(+)
 create mode 100644 doc/manual/command-ref/nix-copy-closure.md
 create mode 100644 doc/manual/introduction/about-nix.md
 create mode 100644 doc/manual/introduction/quick-start.md
 create mode 100644 doc/manual/manual.md

diff --git a/doc/manual/command-ref/nix-copy-closure.md b/doc/manual/command-ref/nix-copy-closure.md
new file mode 100644
index 000000000..037334c4d
--- /dev/null
+++ b/doc/manual/command-ref/nix-copy-closure.md
@@ -0,0 +1,78 @@
+Title: nix-copy-closure
+
+# Name
+
+`nix-copy-closure` - copy a closure to or from a remote machine via SSH
+
+# Synopsis
+
+`nix-copy-closure` [`--to` | `--from`] [`--gzip`] [`--include-outputs`] [`--use-substitutes` | `-s`] [`-v`] _user@machine_ _paths_
+
+# Description
+
+`nix-copy-closure` gives you an easy and efficient way to exchange
+software between machines.  Given one or more Nix store _paths_ on the
+local machine, `nix-copy-closure` computes the closure of those paths
+(i.e. all their dependencies in the Nix store), and copies all paths
+in the closure to the remote machine via the `ssh` (Secure Shell)
+command.  With the `--from` option, the direction is reversed: the
+closure of _paths_ on a remote machine is copied to the Nix store on
+the local machine.
+
+This command is efficient because it only sends the store paths
+that are missing on the target machine.
+
+Since `nix-copy-closure` calls `ssh`, you may be asked to type in the
+appropriate password or passphrase.  In fact, you may be asked _twice_
+because `nix-copy-closure` currently connects twice to the remote
+machine, first to get the set of paths missing on the target machine,
+and second to send the dump of those paths.  If this bothers you, use
+`ssh-agent`.
+
+# Options
+
+*`--to`*
+: Copy the closure of _paths_ from the local Nix store to the Nix
+  store on _machine_. This is the default.
+
+*`--from`*
+: Copy the closure of _paths_ from the Nix store on _machine_ to the
+  local Nix store.
+
+*`--gzip`*
+: Enable compression of the SSH connection.
+
+*`--include-outputs`*
+: Also copy the outputs of store derivations included in the closure.
+
+*`--use-substitutes` / `-s`*
+: Attempt to download missing paths on the target machine using Nix’s
+  substitute mechanism.  Any paths that cannot be substituted on the
+  target are still copied normally from the source.  This is useful,
+  for instance, if the connection between the source and target
+  machine is slow, but the connection between the target machine and
+  `nixos.org` (the default binary cache server) is
+  fast.
+
+*`-v`*
+: Show verbose output.
+
+# Environment variables
+
+*`NIX_SSHOPTS`*
+: Additional options to be passed to `ssh` on the command
+  line.
+
+# Examples
+
+Copy Firefox with all its dependencies to a remote machine:
+
+    $ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)
+
+Copy Subversion from a remote machine and then install it into a user
+environment:
+
+    $ nix-copy-closure --from alice@itchy.labs \
+        /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+    $ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+
diff --git a/doc/manual/introduction/about-nix.md b/doc/manual/introduction/about-nix.md
new file mode 100644
index 000000000..b3cd00bd3
--- /dev/null
+++ b/doc/manual/introduction/about-nix.md
@@ -0,0 +1,181 @@
+# About Nix
+
+Nix is a _purely functional package manager_.  This means that it
+treats packages like values in purely functional programming languages
+such as Haskell — they are built by functions that don’t have
+side-effects, and they never change after they have been built.  Nix
+stores packages in the _Nix store_, usually the directory
+`/nix/store`, where each package has its own unique subdirectory such
+as
+
+    /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/
+
+where `b6gvzjyb2pg0…` is a unique identifier for the package that
+captures all its dependencies (it’s a cryptographic hash of the
+package’s build dependency graph).  This enables many powerful
+features.
+
+## Multiple versions
+
+You can have multiple versions or variants of a package
+installed at the same time.  This is especially important when
+different applications have dependencies on different versions of the
+same package — it prevents the “DLL hell”.  Because of the hashing
+scheme, different versions of a package end up in different paths in
+the Nix store, so they don’t interfere with each other.
+
+An important consequence is that operations like upgrading or
+uninstalling an application cannot break other applications, since
+these operations never “destructively” update or delete files that are
+used by other packages.
+
+## Complete dependencies
+
+Nix helps you make sure that package dependency specifications are
+complete.  In general, when you’re making a package for a package
+management system like RPM, you have to specify for each package what
+its dependencies are, but there are no guarantees that this
+specification is complete.  If you forget a dependency, then the
+package will build and work correctly on _your_ machine if you have
+the dependency installed, but not on the end user's machine if it's
+not there.
+
+Since Nix on the other hand doesn’t install packages in “global”
+locations like `/usr/bin` but in package-specific directories, the
+risk of incomplete dependencies is greatly reduced.  This is because
+tools such as compilers don’t search in per-packages directories such
+as `/nix/store/5lbfaxb722zp…-openssl-0.9.8d/include`, so if a package
+builds correctly on your system, this is because you specified the
+dependency explicitly. This takes care of the build-time dependencies.
+
+Once a package is built, runtime dependencies are found by scanning
+binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`).
+This sounds risky, but it works extremely well.
+
+## Multi-user support
+
+Nix has multi-user support.  This means that non-privileged users can
+securely install software.  Each user can have a different _profile_,
+a set of packages in the Nix store that appear in the user’s `PATH`.
+If a user installs a package that another user has already installed
+previously, the package won’t be built or downloaded a second time.
+At the same time, it is not possible for one user to inject a Trojan
+horse into a package that might be used by another user.
+
+## Atomic upgrades and rollbacks
+
+Since package management operations never overwrite packages in the
+Nix store but just add new versions in different paths, they are
+_atomic_.  So during a package upgrade, there is no time window in
+which the package has some files from the old version and some files
+from the new version — which would be bad because a program might well
+crash if it’s started during that period.
+
+And since packages aren’t overwritten, the old versions are still
+there after an upgrade.  This means that you can _roll back_ to the
+old version:
+
+    $ nix-env --upgrade some-packages
+    $ nix-env --rollback
+
+## Garbage collection
+
+When you uninstall a package like this…
+
+    $ nix-env --uninstall firefox
+
+the package isn’t deleted from the system right away (after all, you
+might want to do a rollback, or it might be in the profiles of other
+users).  Instead, unused packages can be deleted safely by running the
+_garbage collector_:
+
+    $ nix-collect-garbage
+
+This deletes all packages that aren’t in use by any user profile or by
+a currently running program.
+
+## Functional package language
+
+Packages are built from _Nix expressions_, which is a simple
+functional language.  A Nix expression describes everything that goes
+into a package build action (a “derivation”): other packages, sources,
+the build script, environment variables for the build script, etc.
+Nix tries very hard to ensure that Nix expressions are
+_deterministic_: building a Nix expression twice should yield the same
+result.
+
+Because it’s a functional language, it’s easy to support
+building variants of a package: turn the Nix expression into a
+function and call it any number of times with the appropriate
+arguments.  Due to the hashing scheme, variants don’t conflict with
+each other in the Nix store.
+
+## Transparent source/binary deployment
+
+Nix expressions generally describe how to build a package from
+source, so an installation action like
+
+    $ nix-env --install firefox
+
+_could_ cause quite a bit of build activity, as not only Firefox but
+also all its dependencies (all the way up to the C library and the
+compiler) would have to built, at least if they are not already in the
+Nix store.  This is a _source deployment model_.  For most users,
+building from source is not very pleasant as it takes far too long.
+However, Nix can automatically skip building from source and instead
+use a _binary cache_, a web server that provides pre-built
+binaries. For instance, when asked to build
+`/nix/store/b6gvzjyb2pg0…-firefox-33.1` from source, Nix would first
+check if the file `https://cache.nixos.org/b6gvzjyb2pg0….narinfo`
+exists, and if so, fetch the pre-built binary referenced from there;
+otherwise, it would fall back to building from source.
+
+## Nix Packages collection
+
+We provide a large set of Nix expressions containing hundreds of
+existing Unix packages, the _Nix Packages collection_ (Nixpkgs).
+
+## Managing build environments
+
+Nix is extremely useful for developers as it makes it easy to
+automatically set up the build environment for a package. Given a Nix
+expression that describes the dependencies of your package, the
+command `nix-shell` will build or download those dependencies if
+they’re not already in your Nix store, and then start a Bash shell in
+which all necessary environment variables (such as compiler search
+paths) are set.
+
+For example, the following command gets all dependencies of the
+Pan newsreader, as described by [its
+Nix expression](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix):
+
+    $ nix-shell '<nixpkgs>' -A pan
+
+You’re then dropped into a shell where you can edit, build and test
+the package:
+
+    [nix-shell]$ tar xf $src
+    [nix-shell]$ cd pan-*
+    [nix-shell]$ ./configure
+    [nix-shell]$ make
+    [nix-shell]$ ./pan/gui/pan
+
+## Portability
+
+Nix runs on Linux and macOS.
+
+## NixOS
+
+NixOS is a Linux distribution based on Nix.  It uses Nix not just for
+package management but also to manage the system configuration (e.g.,
+to build configuration files in `/etc`).  This means, among other
+things, that it is easy to roll back the entire configuration of the
+system to an earlier state.  Also, users can install software without
+root privileges.  For more information and downloads, see the [NixOS
+homepage](https://nixos.org/).
+
+## License
+
+Nix is released under the terms of the [GNU LGPLv2.1 or (at your
+option) any later
+version](http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html).
diff --git a/doc/manual/introduction/quick-start.md b/doc/manual/introduction/quick-start.md
new file mode 100644
index 000000000..21c03e3cf
--- /dev/null
+++ b/doc/manual/introduction/quick-start.md
@@ -0,0 +1,79 @@
+# Quick Start
+
+This chapter is for impatient people who don't like reading
+documentation.  For more in-depth information you are kindly referred
+to subsequent chapters.
+
+1. Install single-user Nix by running the following:
+
+        $ bash <(curl -L https://nixos.org/nix/install)
+
+   This will install Nix in `/nix`. The install script will create
+   `/nix` using `sudo`, so make sure you have sufficient rights.  (For
+   other installation methods, see
+   [here](../installation/installation.md).)
+
+1. See what installable packages are currently available in the
+   channel:
+
+        $ nix-env -qa
+        docbook-xml-4.3
+        docbook-xml-4.5
+        firefox-33.0.2
+        hello-2.9
+        libxslt-1.1.28
+        …
+
+1. Install some packages from the channel:
+
+        $ nix-env -i hello
+
+   This should download pre-built packages; it should not build them
+   locally (if it does, something went wrong).
+
+1. Test that they work:
+
+        $ which hello
+        /home/eelco/.nix-profile/bin/hello
+        $ hello
+        Hello, world!
+
+1. Uninstall a package:
+
+        $ nix-env -e hello
+
+1. You can also test a package without installing it:
+
+        $ nix-shell -p hello
+
+   This builds or downloads GNU Hello and its dependencies, then drops
+   you into a Bash shell where the `hello` command is present, all
+   without affecting your normal environment:
+
+        [nix-shell:~]$ hello
+        Hello, world!
+
+        [nix-shell:~]$ exit
+
+        $ hello
+        hello: command not found
+
+1. To keep up-to-date with the channel, do:
+
+        $ nix-channel --update nixpkgs
+        $ nix-env -u '*'
+
+   The latter command will upgrade each installed package for which
+   there is a “newer” version (as determined by comparing the version
+   numbers).
+
+1. If you're unhappy with the result of a `nix-env` action (e.g., an
+   upgraded package turned out not to work properly), you can go back:
+
+        $ nix-env --rollback
+
+1. You should periodically run the Nix garbage collector to get rid of
+   unused packages, since uninstalls or upgrades don't actually delete
+   them:
+
+        $ nix-collect-garbage -d
diff --git a/doc/manual/manual.md b/doc/manual/manual.md
new file mode 100644
index 000000000..58e37436f
--- /dev/null
+++ b/doc/manual/manual.md
@@ -0,0 +1,8 @@
+Title: Nix Package Manager Guide
+
+1. Introduction
+   1. [About Nix](./introduction/about-nix.md)
+   1. [Quick Start](./introduction/quick-start.md)
+1. Command Reference
+   1. Utilities
+      1. [nix-copy-closure](./command-ref/nix-copy-closure.md)
diff --git a/flake.nix b/flake.nix
index a707e90e7..dfe4ecd49 100644
--- a/flake.nix
+++ b/flake.nix
@@ -66,6 +66,7 @@
             libxslt
             docbook5
             docbook_xsl_ns
+            lowdown
             autoconf-archive
             autoreconfHook
 
@@ -187,6 +188,26 @@
 
         };
 
+        lowdown = with final; stdenv.mkDerivation {
+          name = "lowdown-0.7.1";
+
+          src = fetchurl {
+            url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.1.tar.gz;
+            hash = "sha512-1daoAQfYD0LdhK6aFhrSQvadjc5GsSPBZw0fJDb+BEHYMBLjqiUl2A7H8N+l0W4YfGKqbsPYSrCy4vct+7U6FQ==";
+          };
+
+          outputs = [ "out" "dev" ];
+
+          buildInputs = [ which ];
+
+          configurePhase =
+            ''
+              ./configure \
+                PREFIX=${placeholder "dev"} \
+                BINDIR=${placeholder "out"}/bin
+            '';
+        };
+
       };
 
       hydraJobs = {

From e0ea3c82ca9e46359c55c9f716fec016f8d483ea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 20:27:23 +0200
Subject: [PATCH 018/882] Use mdbook

---
 configure.ac                                  |  2 -
 doc/manual/local.mk                           | 81 +++++--------------
 doc/manual/src/SUMMARY.md                     |  7 ++
 doc/manual/src/command-ref/command-ref.md     |  2 +
 .../{ => src}/command-ref/nix-copy-closure.md |  0
 doc/manual/src/command-ref/utilities.md       |  3 +
 .../about-nix.md => src/introduction.md}      |  2 +-
 .../{introduction => src}/quick-start.md      |  0
 flake.nix                                     |  5 +-
 9 files changed, 32 insertions(+), 70 deletions(-)
 create mode 100644 doc/manual/src/SUMMARY.md
 create mode 100644 doc/manual/src/command-ref/command-ref.md
 rename doc/manual/{ => src}/command-ref/nix-copy-closure.md (100%)
 create mode 100644 doc/manual/src/command-ref/utilities.md
 rename doc/manual/{introduction/about-nix.md => src/introduction.md} (99%)
 rename doc/manual/{introduction => src}/quick-start.md (100%)

diff --git a/configure.ac b/configure.ac
index 2f29cf864..eecb107d7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -117,8 +117,6 @@ fi
 ])
 
 NEED_PROG(bash, bash)
-AC_PATH_PROG(xmllint, xmllint, false)
-AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 AC_PATH_PROG(dot, dot)
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index ce05c6234..a91d497ce 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -1,84 +1,39 @@
-
 ifeq ($(doc_generate),yes)
 
-XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
-  --param section.autolabel 1 \
-  --param section.label.includes.component.label 1 \
-  --param xref.with.number.and.title 1 \
-  --param toc.section.depth 3 \
-  --param admon.style \'\' \
-  --param callout.graphics 0 \
-  --param contrib.inline.enabled 0 \
-  --stringparam generate.toc "book toc" \
-  --param keep.relative.image.uris 0
+MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
 
-docbookxsl = http://docbook.sourceforge.net/release/xsl-ns/current
-docbookrng = http://docbook.org/xml/5.0/rng/docbook.rng
+#$(d)/version.txt:
+#	$(trace-gen) echo -n $(PACKAGE_VERSION) > $@
 
-MANUAL_SRCS := $(call rwildcard, $(d), *.xml)
+clean-files += $(d)/version.txt
 
-
-# Do XInclude processing / RelaxNG validation
-$(d)/manual.xmli: $(d)/manual.xml $(MANUAL_SRCS) $(d)/version.txt
-	$(trace-gen) $(xmllint) --nonet --xinclude $< -o $@.tmp
-	@mv $@.tmp $@
-
-$(d)/version.txt:
-	$(trace-gen) echo -n $(PACKAGE_VERSION) > $@
-
-# Note: RelaxNG validation requires xmllint >= 2.7.4.
-$(d)/manual.is-valid: $(d)/manual.xmli
-	$(trace-gen) $(XSLTPROC) --novalid --stringparam profile.condition manual \
-	  $(docbookxsl)/profiling/profile.xsl $< 2> /dev/null | \
-	  $(xmllint) --nonet --noout --relaxng $(docbookrng) -
-	@touch $@
-
-clean-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid
-
-dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid
+dist-files += $(d)/version.txt
 
 
 # Generate man pages.
 man-pages := $(foreach n, \
-  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
-  nix-collect-garbage.1 \
-  nix-prefetch-url.1 nix-channel.1 \
-  nix-hash.1 nix-copy-closure.1 \
-  nix.conf.5 nix-daemon.8, \
+  nix-copy-closure.1, \
   $(d)/$(n))
-
-$(firstword $(man-pages)): $(d)/manual.xmli $(d)/manual.is-valid
-	$(trace-gen) $(XSLTPROC) --novalid --stringparam profile.condition manpage \
-	  $(docbookxsl)/profiling/profile.xsl $< 2> /dev/null | \
-	  (cd doc/manual && $(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl -)
-
-$(wordlist 2, $(words $(man-pages)), $(man-pages)): $(firstword $(man-pages))
+#  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
+#  nix-collect-garbage.1, \
+#  nix-prefetch-url.1 nix-channel.1 \
+#  nix-hash.1 nix-copy-closure.1 \
+#  nix.conf.5 nix-daemon.8, \
 
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 dist-files += $(man-pages)
 
+$(d)/nix-copy-closure.1: $(d)/src/command-ref/nix-copy-closure.md
+
+%.1: %.md
+	$(trace-gen) lowdown -sT man $^ -o $@
 
 # Generate the HTML manual.
-$(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
-	$(trace-gen) $(XSLTPROC) --xinclude --stringparam profile.condition manual \
-	  $(docbookxsl)/profiling/profile.xsl $< | \
-	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
+install: $(docdir)/manual/index.html
 
-$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
-
-$(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
-
-$(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
-
-
-all: $(d)/manual.html
-
-
-
-clean-files += $(d)/manual.html
-
-dist-files += $(d)/manual.html
+$(docdir)/manual/index.html: $(MANUAL_SRCS)
+	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
 
 
 endif
diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
new file mode 100644
index 000000000..6897f70d9
--- /dev/null
+++ b/doc/manual/src/SUMMARY.md
@@ -0,0 +1,7 @@
+# Table of Contents
+
+- [Introduction](./introduction.md)
+- [Quick Start](./quick-start.md)
+- [Command Reference](./command-ref/command-ref.md)
+  - [Utilities](./command-ref/utilities.md)
+    - [nix-copy-closure](./command-ref/nix-copy-closure.md)
diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/command-ref.md
new file mode 100644
index 000000000..b15a50a3b
--- /dev/null
+++ b/doc/manual/src/command-ref/command-ref.md
@@ -0,0 +1,2 @@
+This section lists commands and options that you can use when you
+work with Nix.
diff --git a/doc/manual/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
similarity index 100%
rename from doc/manual/command-ref/nix-copy-closure.md
rename to doc/manual/src/command-ref/nix-copy-closure.md
diff --git a/doc/manual/src/command-ref/utilities.md b/doc/manual/src/command-ref/utilities.md
new file mode 100644
index 000000000..5ba8a02a3
--- /dev/null
+++ b/doc/manual/src/command-ref/utilities.md
@@ -0,0 +1,3 @@
+# Utilities
+
+This section lists utilities that you can use when you work with Nix.
diff --git a/doc/manual/introduction/about-nix.md b/doc/manual/src/introduction.md
similarity index 99%
rename from doc/manual/introduction/about-nix.md
rename to doc/manual/src/introduction.md
index b3cd00bd3..b54b0d02d 100644
--- a/doc/manual/introduction/about-nix.md
+++ b/doc/manual/src/introduction.md
@@ -1,4 +1,4 @@
-# About Nix
+# Introduction
 
 Nix is a _purely functional package manager_.  This means that it
 treats packages like values in purely functional programming languages
diff --git a/doc/manual/introduction/quick-start.md b/doc/manual/src/quick-start.md
similarity index 100%
rename from doc/manual/introduction/quick-start.md
rename to doc/manual/src/quick-start.md
diff --git a/flake.nix b/flake.nix
index dfe4ecd49..cebbdf9d9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -62,10 +62,7 @@
         buildDeps =
           [ bison
             flex
-            libxml2
-            libxslt
-            docbook5
-            docbook_xsl_ns
+            mdbook
             lowdown
             autoconf-archive
             autoreconfHook

From 8e41c388679aedc12eb1a7a7aff1b4818040ff4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 20:46:12 +0200
Subject: [PATCH 019/882] Remove references to xmllint

---
 Makefile.config.in   | 2 --
 doc/manual/local.mk  | 2 --
 tests/common.sh.in   | 1 -
 tests/nix-channel.sh | 6 ------
 4 files changed, 11 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 5c245b8e9..3845b3be0 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -41,5 +41,3 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-xmllint = @xmllint@
-xsltproc = @xsltproc@
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index a91d497ce..65acd658c 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -25,8 +25,6 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 dist-files += $(man-pages)
 
 $(d)/nix-copy-closure.1: $(d)/src/command-ref/nix-copy-closure.md
-
-%.1: %.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
 # Generate the HTML manual.
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 308126094..5e00d64f1 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -32,7 +32,6 @@ export PATH=@bindir@:$PATH
 coreutils=@coreutils@
 
 export dot=@dot@
-export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index 49c68981a..63c0f97ba 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -28,9 +28,6 @@ nix-channel --update
 
 # Do a query.
 nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml
-if [ "$xmllint" != false ]; then
-    $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
-fi
 grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
 grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml
 
@@ -47,9 +44,6 @@ nix-channel --update
 
 # Do a query.
 nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml
-if [ "$xmllint" != false ]; then
-    $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
-fi
 grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
 grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml
 

From ebdc1f6dfbc4998dd974ee58bd66b51130b5abde Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 23:03:13 +0200
Subject: [PATCH 020/882] Typo

---
 doc/manual/hacking.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml
index d25d4b84a..5d9f08d38 100644
--- a/doc/manual/hacking.xml
+++ b/doc/manual/hacking.xml
@@ -32,7 +32,7 @@ i686-linux, x86_64-darwin, x86_64-linux.
 i.e.
 
 <screen>
-nix-build -A defaultPackage.x86_64-linux
+$ nix-build -A defaultPackage.x86_64-linux
 </screen>
 
 </para>

From 315407c16f16c3f1d6c5edc087b198c3b68d1dff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 23:03:28 +0200
Subject: [PATCH 021/882] Remove subtitles

---
 doc/manual/advanced-topics/diff-hook.xml       | 5 +----
 doc/manual/advanced-topics/post-build-hook.xml | 2 --
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml
index f01ab71b3..85cfdfc02 100644
--- a/doc/manual/advanced-topics/diff-hook.xml
+++ b/doc/manual/advanced-topics/diff-hook.xml
@@ -5,10 +5,7 @@
       version="5.0"
       >
 
-<title>Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option></title>
-
-<subtitle>Check build reproducibility by running builds multiple times
-and comparing their results.</subtitle>
+<title>Verifying Build Reproducibility</title>
 
 <para>Specify a program with Nix's <xref linkend="conf-diff-hook" /> to
 compare build results when two builds produce different results. Note:
diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
index 6cc286ee1..1480ab86a 100644
--- a/doc/manual/advanced-topics/post-build-hook.xml
+++ b/doc/manual/advanced-topics/post-build-hook.xml
@@ -6,8 +6,6 @@
       >
 
 <title>Using the <option linkend="conf-post-build-hook">post-build-hook</option></title>
-<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
-
 
 <section xml:id="chap-post-build-hook-caveats">
   <title>Implementation Caveats</title>

From d004715665046ff424f267deccccb78c9d5cabb7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 23:03:53 +0200
Subject: [PATCH 022/882] Fix link

---
 doc/manual/src/quick-start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 21c03e3cf..3c519217b 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -11,7 +11,7 @@ to subsequent chapters.
    This will install Nix in `/nix`. The install script will create
    `/nix` using `sudo`, so make sure you have sufficient rights.  (For
    other installation methods, see
-   [here](../installation/installation.md).)
+   [here](installation/installation.md).)
 
 1. See what installable packages are currently available in the
    channel:

From ef606760abd87c98371fbc08c1f25ad897823a2a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 23:17:48 +0200
Subject: [PATCH 023/882] Pandoc conversion

---
 doc/manual/manual.md                          |   8 -
 doc/manual/src/SUMMARY.md                     |  90 +-
 .../src/advanced-topics/advanced-topics.md    |   1 +
 .../src/advanced-topics/cores-vs-jobs.md      |  42 +
 doc/manual/src/advanced-topics/diff-hook.md   | 141 +++
 .../src/advanced-topics/distributed-builds.md | 141 +++
 .../src/advanced-topics/post-build-hook.md    | 113 +++
 .../src/expressions/advanced-attributes.md    | 233 +++++
 .../src/expressions/arguments-variables.md    |  72 ++
 doc/manual/src/expressions/build-script.md    |  72 ++
 doc/manual/src/expressions/builder-syntax.md  |  72 ++
 doc/manual/src/expressions/builtins.md        | 839 ++++++++++++++++++
 doc/manual/src/expressions/derivations.md     | 154 ++++
 .../src/expressions/expression-language.md    |  12 +
 .../src/expressions/expression-syntax.md      |  91 ++
 doc/manual/src/expressions/generic-builder.md |  61 ++
 .../src/expressions/language-constructs.md    | 291 ++++++
 .../src/expressions/language-operators.md     |  32 +
 doc/manual/src/expressions/language-values.md | 209 +++++
 .../expressions/simple-building-testing.md    |  57 ++
 .../src/expressions/simple-expression.md      |  27 +
 .../expressions/writing-nix-expressions.md    |  12 +
 doc/manual/src/glossary.md                    | 102 +++
 doc/manual/src/hacking.md                     |  47 +
 .../src/installation/building-source.md       |  34 +
 doc/manual/src/installation/env-variables.md  |  56 ++
 doc/manual/src/installation/installation.md   |   2 +
 .../src/installation/installing-binary.md     | 273 ++++++
 .../src/installation/installing-source.md     |   4 +
 doc/manual/src/installation/multi-user.md     |  63 ++
 doc/manual/src/installation/nix-security.md   |  15 +
 .../src/installation/obtaining-source.md      |  16 +
 .../src/installation/prerequisites-source.md  |  80 ++
 doc/manual/src/installation/single-user.md    |   9 +
 .../src/installation/supported-platforms.md   |   7 +
 doc/manual/src/installation/upgrading.md      |  14 +
 .../package-management/basic-package-mgmt.md  | 148 +++
 .../binary-cache-substituter.md               |  42 +
 doc/manual/src/package-management/channels.md |  42 +
 .../src/package-management/copy-closure.md    |  34 +
 .../package-management/garbage-collection.md  |  61 ++
 .../garbage-collector-roots.md                |  16 +
 .../package-management/package-management.md  |   4 +
 doc/manual/src/package-management/profiles.md | 119 +++
 .../src/package-management/s3-substituter.md  | 133 +++
 .../package-management/sharing-packages.md    |   6 +
 .../src/package-management/ssh-substituter.md |  52 ++
 doc/manual/src/release-notes/release-notes.md |   1 +
 doc/manual/src/release-notes/rl-0.10.1.md     |   5 +
 doc/manual/src/release-notes/rl-0.10.md       | 212 +++++
 doc/manual/src/release-notes/rl-0.11.md       | 167 ++++
 doc/manual/src/release-notes/rl-0.12.md       | 123 +++
 doc/manual/src/release-notes/rl-0.13.md       |  55 ++
 doc/manual/src/release-notes/rl-0.14.md       |  21 +
 doc/manual/src/release-notes/rl-0.15.md       |   5 +
 doc/manual/src/release-notes/rl-0.16.md       |  25 +
 doc/manual/src/release-notes/rl-0.5.md        |   3 +
 doc/manual/src/release-notes/rl-0.6.md        |  64 ++
 doc/manual/src/release-notes/rl-0.7.md        |  21 +
 doc/manual/src/release-notes/rl-0.8.1.md      |   8 +
 doc/manual/src/release-notes/rl-0.8.md        | 166 ++++
 doc/manual/src/release-notes/rl-0.9.1.md      |   4 +
 doc/manual/src/release-notes/rl-0.9.2.md      |  11 +
 doc/manual/src/release-notes/rl-0.9.md        |  72 ++
 doc/manual/src/release-notes/rl-1.0.md        |  68 ++
 doc/manual/src/release-notes/rl-1.1.md        |  61 ++
 doc/manual/src/release-notes/rl-1.10.md       |  31 +
 doc/manual/src/release-notes/rl-1.11.10.md    |  21 +
 doc/manual/src/release-notes/rl-1.11.md       |  87 ++
 doc/manual/src/release-notes/rl-1.2.md        |  97 ++
 doc/manual/src/release-notes/rl-1.3.md        |  10 +
 doc/manual/src/release-notes/rl-1.4.md        |  22 +
 doc/manual/src/release-notes/rl-1.5.1.md      |   4 +
 doc/manual/src/release-notes/rl-1.5.2.md      |   4 +
 doc/manual/src/release-notes/rl-1.5.md        |   4 +
 doc/manual/src/release-notes/rl-1.6.1.md      |  32 +
 doc/manual/src/release-notes/rl-1.6.md        |  72 ++
 doc/manual/src/release-notes/rl-1.7.md        | 140 +++
 doc/manual/src/release-notes/rl-1.8.md        |  88 ++
 doc/manual/src/release-notes/rl-1.9.md        | 143 +++
 doc/manual/src/release-notes/rl-2.0.md        | 557 ++++++++++++
 doc/manual/src/release-notes/rl-2.1.md        |  49 +
 doc/manual/src/release-notes/rl-2.2.md        |  82 ++
 doc/manual/src/release-notes/rl-2.3.md        |  44 +
 84 files changed, 6715 insertions(+), 13 deletions(-)
 delete mode 100644 doc/manual/manual.md
 create mode 100644 doc/manual/src/advanced-topics/advanced-topics.md
 create mode 100644 doc/manual/src/advanced-topics/cores-vs-jobs.md
 create mode 100644 doc/manual/src/advanced-topics/diff-hook.md
 create mode 100644 doc/manual/src/advanced-topics/distributed-builds.md
 create mode 100644 doc/manual/src/advanced-topics/post-build-hook.md
 create mode 100644 doc/manual/src/expressions/advanced-attributes.md
 create mode 100644 doc/manual/src/expressions/arguments-variables.md
 create mode 100644 doc/manual/src/expressions/build-script.md
 create mode 100644 doc/manual/src/expressions/builder-syntax.md
 create mode 100644 doc/manual/src/expressions/builtins.md
 create mode 100644 doc/manual/src/expressions/derivations.md
 create mode 100644 doc/manual/src/expressions/expression-language.md
 create mode 100644 doc/manual/src/expressions/expression-syntax.md
 create mode 100644 doc/manual/src/expressions/generic-builder.md
 create mode 100644 doc/manual/src/expressions/language-constructs.md
 create mode 100644 doc/manual/src/expressions/language-operators.md
 create mode 100644 doc/manual/src/expressions/language-values.md
 create mode 100644 doc/manual/src/expressions/simple-building-testing.md
 create mode 100644 doc/manual/src/expressions/simple-expression.md
 create mode 100644 doc/manual/src/expressions/writing-nix-expressions.md
 create mode 100644 doc/manual/src/glossary.md
 create mode 100644 doc/manual/src/hacking.md
 create mode 100644 doc/manual/src/installation/building-source.md
 create mode 100644 doc/manual/src/installation/env-variables.md
 create mode 100644 doc/manual/src/installation/installation.md
 create mode 100644 doc/manual/src/installation/installing-binary.md
 create mode 100644 doc/manual/src/installation/installing-source.md
 create mode 100644 doc/manual/src/installation/multi-user.md
 create mode 100644 doc/manual/src/installation/nix-security.md
 create mode 100644 doc/manual/src/installation/obtaining-source.md
 create mode 100644 doc/manual/src/installation/prerequisites-source.md
 create mode 100644 doc/manual/src/installation/single-user.md
 create mode 100644 doc/manual/src/installation/supported-platforms.md
 create mode 100644 doc/manual/src/installation/upgrading.md
 create mode 100644 doc/manual/src/package-management/basic-package-mgmt.md
 create mode 100644 doc/manual/src/package-management/binary-cache-substituter.md
 create mode 100644 doc/manual/src/package-management/channels.md
 create mode 100644 doc/manual/src/package-management/copy-closure.md
 create mode 100644 doc/manual/src/package-management/garbage-collection.md
 create mode 100644 doc/manual/src/package-management/garbage-collector-roots.md
 create mode 100644 doc/manual/src/package-management/package-management.md
 create mode 100644 doc/manual/src/package-management/profiles.md
 create mode 100644 doc/manual/src/package-management/s3-substituter.md
 create mode 100644 doc/manual/src/package-management/sharing-packages.md
 create mode 100644 doc/manual/src/package-management/ssh-substituter.md
 create mode 100644 doc/manual/src/release-notes/release-notes.md
 create mode 100644 doc/manual/src/release-notes/rl-0.10.1.md
 create mode 100644 doc/manual/src/release-notes/rl-0.10.md
 create mode 100644 doc/manual/src/release-notes/rl-0.11.md
 create mode 100644 doc/manual/src/release-notes/rl-0.12.md
 create mode 100644 doc/manual/src/release-notes/rl-0.13.md
 create mode 100644 doc/manual/src/release-notes/rl-0.14.md
 create mode 100644 doc/manual/src/release-notes/rl-0.15.md
 create mode 100644 doc/manual/src/release-notes/rl-0.16.md
 create mode 100644 doc/manual/src/release-notes/rl-0.5.md
 create mode 100644 doc/manual/src/release-notes/rl-0.6.md
 create mode 100644 doc/manual/src/release-notes/rl-0.7.md
 create mode 100644 doc/manual/src/release-notes/rl-0.8.1.md
 create mode 100644 doc/manual/src/release-notes/rl-0.8.md
 create mode 100644 doc/manual/src/release-notes/rl-0.9.1.md
 create mode 100644 doc/manual/src/release-notes/rl-0.9.2.md
 create mode 100644 doc/manual/src/release-notes/rl-0.9.md
 create mode 100644 doc/manual/src/release-notes/rl-1.0.md
 create mode 100644 doc/manual/src/release-notes/rl-1.1.md
 create mode 100644 doc/manual/src/release-notes/rl-1.10.md
 create mode 100644 doc/manual/src/release-notes/rl-1.11.10.md
 create mode 100644 doc/manual/src/release-notes/rl-1.11.md
 create mode 100644 doc/manual/src/release-notes/rl-1.2.md
 create mode 100644 doc/manual/src/release-notes/rl-1.3.md
 create mode 100644 doc/manual/src/release-notes/rl-1.4.md
 create mode 100644 doc/manual/src/release-notes/rl-1.5.1.md
 create mode 100644 doc/manual/src/release-notes/rl-1.5.2.md
 create mode 100644 doc/manual/src/release-notes/rl-1.5.md
 create mode 100644 doc/manual/src/release-notes/rl-1.6.1.md
 create mode 100644 doc/manual/src/release-notes/rl-1.6.md
 create mode 100644 doc/manual/src/release-notes/rl-1.7.md
 create mode 100644 doc/manual/src/release-notes/rl-1.8.md
 create mode 100644 doc/manual/src/release-notes/rl-1.9.md
 create mode 100644 doc/manual/src/release-notes/rl-2.0.md
 create mode 100644 doc/manual/src/release-notes/rl-2.1.md
 create mode 100644 doc/manual/src/release-notes/rl-2.2.md
 create mode 100644 doc/manual/src/release-notes/rl-2.3.md

diff --git a/doc/manual/manual.md b/doc/manual/manual.md
deleted file mode 100644
index 58e37436f..000000000
--- a/doc/manual/manual.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Title: Nix Package Manager Guide
-
-1. Introduction
-   1. [About Nix](./introduction/about-nix.md)
-   1. [Quick Start](./introduction/quick-start.md)
-1. Command Reference
-   1. Utilities
-      1. [nix-copy-closure](./command-ref/nix-copy-closure.md)
diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 6897f70d9..f562d1db4 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -1,7 +1,87 @@
 # Table of Contents
 
-- [Introduction](./introduction.md)
-- [Quick Start](./quick-start.md)
-- [Command Reference](./command-ref/command-ref.md)
-  - [Utilities](./command-ref/utilities.md)
-    - [nix-copy-closure](./command-ref/nix-copy-closure.md)
+- [Introduction](introduction.md)
+- [Quick Start](quick-start.md)
+- [Installation](installation/installation.md)
+  - [Supported Platforms](installation/supported-platforms.md)
+  - [Installing a Binary Distribution](installation/installing-binary.md)
+  - [Installing Nix from Source](installation/installing-source.md)
+    - [Prerequisites](installation/prerequisites-source.md)
+    - [Obtaining a Source Distribution](installation/obtaining-source.md)
+    - [Building Nix from Source](installation/building-source.md)
+  - [Security](installation/nix-security.md)
+    - [Single-User Mode](installation/single-user.md)
+    - [Multi-User Mode](installation/multi-user.md)
+  - [Environment Variables](installation/env-variables.md)
+  - [Upgrading Nix](installation/upgrading.md)
+- [Package Management](package-management/package-management.md)
+  - [Basic Package Management](package-management/basic-package-mgmt.md)
+  - [Profiles](package-management/profiles.md)
+  - [Garbage Collection](package-management/garbage-collection.md)
+    - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
+  - [Channels](package-management/channels.md)
+  - [Sharing Packages Between Machines](package-management/sharing-packages.md)
+    - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
+    - [Copying Closures Via SSH](package-management/copy-closure.md)
+    - [Serving a Nix store via SSH](package-management/ssh-substituter.md)
+    - [Serving a Nix store via AWS S3 or S3-compatible Service](package-management/s3-substituter.md)
+- [Writing Nix Expressions](expressions/writing-nix-expressions.md)
+  - [A Simple Nix Expression](expressions/simple-expression.md)
+    - [Expression Syntax](expressions/expression-syntax.md)
+    - [Build Script](expressions/build-script.md)
+    - [Arguments and Variables](expressions/arguments-variables.md)
+    - [Building and Testing](expressions/simple-building-testing.md)
+    - [Generic Builder Syntax](expressions/generic-builder.md)
+  - [Writing Nix Expressions](expressions/expression-language.md)
+    - [Values](expressions/language-values.md)
+    - [Language Constructs](expressions/language-constructs.md)
+    - [Operators](expressions/language-operators.md)
+    - [Derivations](expressions/derivations.md)
+      - [Advanced Attributes](expressions/advanced-attributes.md)
+    - [Built-in Functions](expressions/builtins.md)
+- [Advanced Topics](advanced-topics/advanced-topics.md)
+  - [Remote Builds](advanced-topics/distributed-builds.md)
+  - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
+  - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
+  - [Using the `post-build-hook`](advanced-topics/post-build-hook.md)
+- [Command Reference](command-ref/command-ref.md)
+  - [Utilities](command-ref/utilities.md)
+    - [nix-copy-closure](command-ref/nix-copy-closure.md)
+- [Glossary](glossary.md)
+- [Hacking](hacking.md)
+- [Release Notes](release-notes/release-notes.md)
+  - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
+  - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)
+  - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md)
+  - [Release 2.0 (2018-02-22)](release-notes/rl-2.0.md)
+  - [Release 1.11.10 (2017-06-12)](release-notes/rl-1.11.10.md)
+  - [Release 1.11 (2016-01-19)](release-notes/rl-1.11.md)
+  - [Release 1.10 (2015-09-03)](release-notes/rl-1.10.md)
+  - [Release 1.9 (2015-06-12)](release-notes/rl-1.9.md)
+  - [Release 1.8 (2014-12-14)](release-notes/rl-1.8.md)
+  - [Release 1.7 (2014-04-11)](release-notes/rl-1.7.md)
+  - [Release 1.6.1 (2013-10-28)](release-notes/rl-1.6.1.md)
+  - [Release 1.6 (2013-09-10)](release-notes/rl-1.6.md)
+  - [Release 1.5.2 (2013-05-13)](release-notes/rl-1.5.2.md)
+  - [Release 1.5 (2013-02-27)](release-notes/rl-1.5.md)
+  - [Release 1.4 (2013-02-26)](release-notes/rl-1.4.md)
+  - [Release 1.3 (2013-01-04)](release-notes/rl-1.3.md)
+  - [Release 1.2 (2012-12-06)](release-notes/rl-1.2.md)
+  - [Release 1.1 (2012-07-18)](release-notes/rl-1.1.md)
+  - [Release 1.0 (2012-05-11)](release-notes/rl-1.0.md)
+  - [Release 0.16 (2010-08-17)](release-notes/rl-0.16.md)
+  - [Release 0.15 (2010-03-17)](release-notes/rl-0.15.md)
+  - [Release 0.14 (2010-02-04)](release-notes/rl-0.14.md)
+  - [Release 0.13 (2009-11-05)](release-notes/rl-0.13.md)
+  - [Release 0.12 (2008-11-20)](release-notes/rl-0.12.md)
+  - [Release 0.11 (2007-12-31)](release-notes/rl-0.11.md)
+  - [Release 0.10.1 (2006-10-11)](release-notes/rl-0.10.1.md)
+  - [Release 0.10 (2006-10-06)](release-notes/rl-0.10.md)
+  - [Release 0.9.2 (2005-09-21)](release-notes/rl-0.9.2.md)
+  - [Release 0.9.1 (2005-09-20)](release-notes/rl-0.9.1.md)
+  - [Release 0.9 (2005-09-16)](release-notes/rl-0.9.md)
+  - [Release 0.8.1 (2005-04-13)](release-notes/rl-0.8.1.md)
+  - [Release 0.8 (2005-04-11)](release-notes/rl-0.8.md)
+  - [Release 0.7 (2005-01-12)](release-notes/rl-0.7.md)
+  - [Release 0.6 (2004-11-14)](release-notes/rl-0.6.md)
+  - [Release 0.5 and earlier](release-notes/rl-0.5.md)
diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/advanced-topics.md
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/doc/manual/src/advanced-topics/advanced-topics.md
@@ -0,0 +1 @@
+
diff --git a/doc/manual/src/advanced-topics/cores-vs-jobs.md b/doc/manual/src/advanced-topics/cores-vs-jobs.md
new file mode 100644
index 000000000..846b6356e
--- /dev/null
+++ b/doc/manual/src/advanced-topics/cores-vs-jobs.md
@@ -0,0 +1,42 @@
+# Tuning Cores and Jobs
+
+Nix has two relevant settings with regards to how your CPU cores will be
+utilized: [???](#conf-cores) and [???](#conf-max-jobs). This chapter
+will talk about what they are, how they interact, and their
+configuration trade-offs.
+
+  - [???](#conf-max-jobs)  
+    Dictates how many separate derivations will be built at the same
+    time. If you set this to zero, the local machine will do no builds.
+    Nix will still substitute from binary caches, and build remotely if
+    remote builders are configured.
+
+  - [???](#conf-cores)  
+    Suggests how many cores each derivation should use. Similar to `make
+    -j`.
+
+The [???](#conf-cores) setting determines the value of
+NIX\_BUILD\_CORES. NIX\_BUILD\_CORES is equal to [???](#conf-cores),
+unless [???](#conf-cores) equals `0`, in which case NIX\_BUILD\_CORES
+will be the total number of cores in the system.
+
+The maximum number of consumed cores is a simple multiplication,
+[???](#conf-max-jobs) \* NIX\_BUILD\_CORES.
+
+The balance on how to set these two independent variables depends upon
+each builder's workload and hardware. Here are a few example scenarios
+on a machine with 24 cores:
+
+| [???](#conf-max-jobs) | [???](#conf-cores) | NIX\_BUILD\_CORES | Maximum Processes | Result                                                                                                                                                                                                                                                                                 |
+| --------------------- | ------------------ | ----------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 1                     | 24                 | 24                | 24                | One derivation will be built at a time, each one can use 24 cores. Undersold if a job can’t use 24 cores.                                                                                                                                                                              |
+| 4                     | 6                  | 6                 | 24                | Four derivations will be built at once, each given access to six cores.                                                                                                                                                                                                                |
+| 12                    | 6                  | 6                 | 72                | 12 derivations will be built at once, each given access to six cores. This configuration is over-sold. If all 12 derivations being built simultaneously try to use all six cores, the machine's performance will be degraded due to extensive context switching between the 12 builds. |
+| 24                    | 1                  | 1                 | 24                | 24 derivations can build at the same time, each using a single core. Never oversold, but derivations which require many cores will be very slow to compile.                                                                                                                            |
+| 24                    | 0                  | 24                | 576               | 24 derivations can build at the same time, each using all the available cores of the machine. Very likely to be oversold, and very likely to suffer context switches.                                                                                                                  |
+
+Balancing 24 Build Cores
+
+It is up to the derivations' build script to respect host's requested
+cores-per-build by following the value of the NIX\_BUILD\_CORES
+environment variable.
diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md
new file mode 100644
index 000000000..2c9896fa5
--- /dev/null
+++ b/doc/manual/src/advanced-topics/diff-hook.md
@@ -0,0 +1,141 @@
+# Verifying Build Reproducibility
+
+Specify a program with Nix's [???](#conf-diff-hook) to compare build
+results when two builds produce different results. Note: this hook is
+only executed if the results are not the same, this hook is not used for
+determining if the results are the same.
+
+For purposes of demonstration, we'll use the following Nix file,
+`deterministic.nix` for testing:
+
+    let
+      inherit (import <nixpkgs> {}) runCommand;
+    in {
+      stable = runCommand "stable" {} ''
+        touch $out
+      '';
+    
+      unstable = runCommand "unstable" {} ''
+        echo $RANDOM > $out
+      '';
+    }
+
+Additionally, `nix.conf` contains:
+
+    diff-hook = /etc/nix/my-diff-hook
+    run-diff-hook = true
+
+where `/etc/nix/my-diff-hook` is an executable file containing:
+
+    #!/bin/sh
+    exec >&2
+    echo "For derivation $3:"
+    /run/current-system/sw/bin/diff -r "$1" "$2"
+
+The diff hook is executed by the same user and group who ran the build.
+However, the diff hook does not have write access to the store path just
+built.
+
+# Spot-Checking Build Determinism
+
+Verify a path which already exists in the Nix store by passing `--check`
+to the build command.
+
+If the build passes and is deterministic, Nix will exit with a status
+code of 0:
+
+    $ nix-build ./deterministic.nix -A stable
+    this derivation will be built:
+      /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
+    building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+    /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+    
+    $ nix-build ./deterministic.nix -A stable --check
+    checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+    /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+
+If the build is not deterministic, Nix will exit with a status code of
+1:
+
+    $ nix-build ./deterministic.nix -A unstable
+    this derivation will be built:
+      /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
+    building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+    /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
+    
+    $ nix-build ./deterministic.nix -A unstable --check
+    checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+    error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
+
+In the Nix daemon's log, we will now see:
+
+    For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv:
+    1c1
+    < 8108
+    ---
+    > 30204
+
+Using `--check` with `--keep-failed` will cause Nix to keep the second
+build's output in a special, `.check` path:
+
+    $ nix-build ./deterministic.nix -A unstable --check --keep-failed
+    checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+    note: keeping build directory '/tmp/nix-build-unstable.drv-0'
+    error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check'
+
+In particular, notice the
+`/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check` output. Nix
+has copied the build results to that directory where you can examine it.
+
+> **Note**
+> 
+> Check paths are not protected against garbage collection, and this
+> path will be deleted on the next garbage collection.
+> 
+> The path is guaranteed to be alive for the duration of
+> [???](#conf-diff-hook)'s execution, but may be deleted any time after.
+> 
+> If the comparison is performed as part of automated tooling, please
+> use the diff-hook or author your tooling to handle the case where the
+> build was not deterministic and also a check path does not exist.
+
+`--check` is only usable if the derivation has been built on the system
+already. If the derivation has not been built Nix will fail with the
+error:
+
+    error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible
+
+Run the build without `--check`, and then try with `--check` again.
+
+# Automatic and Optionally Enforced Determinism Verification
+
+Automatically verify every build at build time by executing the build
+multiple times.
+
+Setting [???](#conf-repeat) and [???](#conf-enforce-determinism) in your
+`nix.conf` permits the automated verification of every build Nix
+performs.
+
+The following configuration will run each build three times, and will
+require the build to be deterministic:
+
+    enforce-determinism = true
+    repeat = 2
+
+Setting [???](#conf-enforce-determinism) to false as in the following
+configuration will run the build multiple times, execute the build hook,
+but will allow the build to succeed even if it does not build
+reproducibly:
+
+    enforce-determinism = false
+    repeat = 1
+
+An example output of this configuration:
+
+    $ nix-build ./test.nix -A unstable
+    this derivation will be built:
+      /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
+    building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
+    building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
+    output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
+    /nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
new file mode 100644
index 000000000..197708ee5
--- /dev/null
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -0,0 +1,141 @@
+# Remote Builds
+
+Nix supports remote builds, where a local Nix installation can forward
+Nix builds to other machines. This allows multiple builds to be
+performed in parallel and allows Nix to perform multi-platform builds in
+a semi-transparent way. For instance, if you perform a build for a
+`x86_64-darwin` on an `i686-linux` machine, Nix can automatically
+forward the build to a `x86_64-darwin` machine, if available.
+
+To forward a build to a remote machine, it’s required that the remote
+machine is accessible via SSH and that it has Nix installed. You can
+test whether connecting to the remote Nix instance works, e.g.
+
+    $ nix ping-store --store ssh://mac
+
+will try to connect to the machine named `mac`. It is possible to
+specify an SSH identity file as part of the remote store URI, e.g.
+
+    $ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key
+
+Since builds should be non-interactive, the key should not have a
+passphrase. Alternatively, you can load identities ahead of time into
+`ssh-agent` or `gpg-agent`.
+
+If you get the error
+
+    bash: nix-store: command not found
+    error: cannot connect to 'mac'
+
+then you need to ensure that the PATH of non-interactive login shells
+contains Nix.
+
+> **Warning**
+> 
+> If you are building via the Nix daemon, it is the Nix daemon user
+> account (that is, `root`) that should have SSH access to the remote
+> machine. If you can’t or don’t want to configure `root` to be able to
+> access to remote machine, you can use a private Nix store instead by
+> passing e.g. `--store ~/my-nix`.
+
+The list of remote machines can be specified on the command line or in
+the Nix configuration file. The former is convenient for testing. For
+example, the following command allows you to build a derivation for
+`x86_64-darwin` on a Linux machine:
+
+    $ uname
+    Linux
+    
+    $ nix build \
+      '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+      --builders 'ssh://mac x86_64-darwin'
+    [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
+    
+    $ cat ./result
+    Darwin
+
+It is possible to specify multiple builders separated by a semicolon or
+a newline, e.g.
+
+``` 
+  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
+```
+
+Each machine specification consists of the following elements, separated
+by spaces. Only the first element is required. To leave a field at its
+default, set it to `-`.
+
+1.  The URI of the remote store in the format
+    `ssh://[username@]hostname`, e.g. `ssh://nix@mac` or `ssh://mac`.
+    For backward compatibility, `ssh://` may be omitted. The hostname
+    may be an alias defined in your `~/.ssh/config`.
+
+2.  A comma-separated list of Nix platform type identifiers, such as
+    `x86_64-darwin`. It is possible for a machine to support multiple
+    platform types, e.g., `i686-linux,x86_64-linux`. If omitted, this
+    defaults to the local platform type.
+
+3.  The SSH identity file to be used to log in to the remote machine. If
+    omitted, SSH will use its regular identities.
+
+4.  The maximum number of builds that Nix will execute in parallel on
+    the machine. Typically this should be equal to the number of CPU
+    cores. For instance, the machine `itchy` in the example will execute
+    up to 8 builds in parallel.
+
+5.  The “speed factor”, indicating the relative speed of the machine. If
+    there are multiple machines of the right type, Nix will prefer the
+    fastest, taking load into account.
+
+6.  A comma-separated list of *supported features*. If a derivation has
+    the `requiredSystemFeatures` attribute, then Nix will only perform
+    the derivation on a machine that has the specified features. For
+    instance, the attribute
+    
+        requiredSystemFeatures = [ "kvm" ];
+    
+    will cause the build to be performed on a machine that has the `kvm`
+    feature.
+
+7.  A comma-separated list of *mandatory features*. A machine will only
+    be used to build a derivation if all of the machine’s mandatory
+    features appear in the derivation’s `requiredSystemFeatures`
+    attribute..
+
+For example, the machine specification
+
+    nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
+    nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
+    nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        1 2 kvm benchmark
+
+specifies several machines that can perform `i686-linux` builds.
+However, `poochie` will only do builds that have the attribute
+
+    requiredSystemFeatures = [ "benchmark" ];
+
+or
+
+    requiredSystemFeatures = [ "benchmark" "kvm" ];
+
+`itchy` cannot do builds that require `kvm`, but `scratchy` does support
+such builds. For regular builds, `itchy` will be preferred over
+`scratchy` because it has a higher speed factor.
+
+Remote builders can also be configured in `nix.conf`, e.g.
+
+    builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd
+
+Finally, remote builders can be configured in a separate configuration
+file included in `builders` via the syntax `@file`. For example,
+
+    builders = @/etc/nix/machines
+
+causes the list of machines in `/etc/nix/machines` to be included. (This
+is the default.)
+
+If you want the builders to use caches, you likely want to set the
+option [`builders-use-substitutes`](#conf-builders-use-substitutes) in
+your local `nix.conf`.
+
+To build only on remote builders and disable building on the local
+machine, you can use the option `--max-jobs 0`.
diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md
new file mode 100644
index 000000000..166b57da6
--- /dev/null
+++ b/doc/manual/src/advanced-topics/post-build-hook.md
@@ -0,0 +1,113 @@
+# Using the `post-build-hook`
+
+# Implementation Caveats
+
+Here we use the post-build hook to upload to a binary cache. This is a
+simple and working example, but it is not suitable for all use cases.
+
+The post build hook program runs after each executed build, and blocks
+the build loop. The build loop exits if the hook program fails.
+
+Concretely, this implementation will make Nix slow or unusable when the
+internet is slow or unreliable.
+
+A more advanced implementation might pass the store paths to a
+user-supplied daemon or queue for processing the store paths outside of
+the build loop.
+
+# Prerequisites
+
+This tutorial assumes you have configured an S3-compatible binary cache
+according to the instructions at
+[???](#ssec-s3-substituter-authenticated-writes), and that the `root`
+user's default AWS profile can upload to the bucket.
+
+# Set up a Signing Key
+
+Use `nix-store --generate-binary-cache-key` to create our public and
+private signing keys. We will sign paths with the private key, and
+distribute the public key for verifying the authenticity of the paths.
+
+    # nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
+    # cat /etc/nix/key.public
+    example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+
+Then, add the public key and the cache URL to your `nix.conf`'s
+[???](#conf-trusted-public-keys) and [???](#conf-substituters) like:
+
+    substituters = https://cache.nixos.org/ s3://example-nix-cache
+    trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+
+We will restart the Nix daemon in a later step.
+
+# Implementing the build hook
+
+Write the following script to `/etc/nix/upload-to-cache.sh`:
+
+    #!/bin/sh
+    
+    set -eu
+    set -f # disable globbing
+    export IFS=' '
+    
+    echo "Signing paths" $OUT_PATHS
+    nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+    echo "Uploading paths" $OUT_PATHS
+    exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
+
+> **Note**
+> 
+> The `$OUT_PATHS` variable is a space-separated list of Nix store
+> paths. In this case, we expect and want the shell to perform word
+> splitting to make each output path its own argument to `nix
+> sign-paths`. Nix guarantees the paths will not contain any spaces,
+> however a store path might contain glob characters. The `set -f`
+> disables globbing in the shell.
+
+Then make sure the hook program is executable by the `root` user:
+
+    # chmod +x /etc/nix/upload-to-cache.sh
+
+# Updating Nix Configuration
+
+Edit `/etc/nix/nix.conf` to run our hook, by adding the following
+configuration snippet at the end:
+
+    post-build-hook = /etc/nix/upload-to-cache.sh
+
+Then, restart the `nix-daemon`.
+
+# Testing
+
+Build any derivation, for example:
+
+    $ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)'
+    this derivation will be built:
+      /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
+    building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
+    running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
+    post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+    post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+    /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+
+Then delete the path from the store, and try substituting it from the
+binary cache:
+
+    $ rm ./result
+    $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+
+Now, copy the path back from the cache:
+
+    $ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+    copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
+    warning: you did not specify '--add-root'; the result might be removed by the garbage collector
+    /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
+
+# Conclusion
+
+We now have a Nix installation configured to automatically sign and
+upload every local build to a remote binary cache.
+
+Before deploying this to production, be sure to consider the
+implementation caveats in [Implementation
+Caveats](#chap-post-build-hook-caveats).
diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
new file mode 100644
index 000000000..683b504a7
--- /dev/null
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -0,0 +1,233 @@
+# Advanced Attributes
+
+Derivations can declare some infrequently used optional attributes.
+
+  - `allowedReferences`  
+    The optional attribute `allowedReferences` specifies a list of legal
+    references (dependencies) of the output of the builder. For example,
+    
+        allowedReferences = [];
+    
+    enforces that the output of a derivation cannot have any runtime
+    dependencies on its inputs. To allow an output to have a runtime
+    dependency on itself, use `"out"` as a list item. This is used in
+    NixOS to check that generated files such as initial ramdisks for
+    booting Linux don’t have accidental dependencies on other paths in
+    the Nix store.
+
+  - `allowedRequisites`  
+    This attribute is similar to `allowedReferences`, but it specifies
+    the legal requisites of the whole closure, so all the dependencies
+    recursively. For example,
+    
+        allowedRequisites = [ foobar ];
+    
+    enforces that the output of a derivation cannot have any other
+    runtime dependency than `foobar`, and in addition it enforces that
+    `foobar` itself doesn't introduce any other dependency itself.
+
+  - `disallowedReferences`  
+    The optional attribute `disallowedReferences` specifies a list of
+    illegal references (dependencies) of the output of the builder. For
+    example,
+    
+        disallowedReferences = [ foo ];
+    
+    enforces that the output of a derivation cannot have a direct
+    runtime dependencies on the derivation `foo`.
+
+  - `disallowedRequisites`  
+    This attribute is similar to `disallowedReferences`, but it
+    specifies illegal requisites for the whole closure, so all the
+    dependencies recursively. For example,
+    
+        disallowedRequisites = [ foobar ];
+    
+    enforces that the output of a derivation cannot have any runtime
+    dependency on `foobar` or any other derivation depending recursively
+    on `foobar`.
+
+  - `exportReferencesGraph`  
+    This attribute allows builders access to the references graph of
+    their inputs. The attribute is a list of inputs in the Nix store
+    whose references graph the builder needs to know. The value of this
+    attribute should be a list of pairs `[ name1
+                    path1 name2
+                    path2 ...
+                    ]`. The references graph of each pathN will be stored in a text file
+    nameN in the temporary build directory. The text files have the
+    format used by `nix-store
+                    --register-validity` (with the deriver fields left empty). For
+    example, when the following derivation is built:
+    
+        derivation {
+          ...
+          exportReferencesGraph = [ "libfoo-graph" libfoo ];
+        };
+    
+    the references graph of `libfoo` is placed in the file
+    `libfoo-graph` in the temporary build directory.
+    
+    `exportReferencesGraph` is useful for builders that want to do
+    something with the closure of a store path. Examples include the
+    builders in NixOS that generate the initial ramdisk for booting
+    Linux (a `cpio` archive containing the closure of the boot script)
+    and the ISO-9660 image for the installation CD (which is populated
+    with a Nix store containing the closure of a bootable NixOS
+    configuration).
+
+  - `impureEnvVars`  
+    This attribute allows you to specify a list of environment variables
+    that should be passed from the environment of the calling user to
+    the builder. Usually, the environment is cleared completely when the
+    builder is executed, but with this attribute you can allow specific
+    environment variables to be passed unmodified. For example,
+    `fetchurl` in Nixpkgs has the line
+    
+        impureEnvVars = [ "http_proxy" "https_proxy" ... ];
+    
+    to make it use the proxy server configuration specified by the user
+    in the environment variables http\_proxy and friends.
+    
+    This attribute is only allowed in [fixed-output
+    derivations](#fixed-output-drvs), where impurities such as these are
+    okay since (the hash of) the output is known in advance. It is
+    ignored for all other derivations.
+    
+    > **Warning**
+    > 
+    > `impureEnvVars` implementation takes environment variables from
+    > the current builder process. When a daemon is building its
+    > environmental variables are used. Without the daemon, the
+    > environmental variables come from the environment of the
+    > `nix-build`.
+
+  - `outputHash`; `outputHashAlgo`; `outputHashMode`  
+    These attributes declare that the derivation is a so-called
+    *fixed-output derivation*, which means that a cryptographic hash of
+    the output is already known in advance. When the build of a
+    fixed-output derivation finishes, Nix computes the cryptographic
+    hash of the output and compares it to the hash declared with these
+    attributes. If there is a mismatch, the build fails.
+    
+    The rationale for fixed-output derivations is derivations such as
+    those produced by the `fetchurl` function. This function downloads a
+    file from a given URL. To ensure that the downloaded file has not
+    been modified, the caller must also specify a cryptographic hash of
+    the file. For example,
+    
+        fetchurl {
+          url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
+          sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+        }
+    
+    It sometimes happens that the URL of the file changes, e.g., because
+    servers are reorganised or no longer available. We then must update
+    the call to `fetchurl`, e.g.,
+    
+        fetchurl {
+          url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+          sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+        }
+    
+    If a `fetchurl` derivation was treated like a normal derivation, the
+    output paths of the derivation and *all derivations depending on it*
+    would change. For instance, if we were to change the URL of the
+    Glibc source distribution in Nixpkgs (a package on which almost all
+    other packages depend) massive rebuilds would be needed. This is
+    unfortunate for a change which we know cannot have a real effect as
+    it propagates upwards through the dependency graph.
+    
+    For fixed-output derivations, on the other hand, the name of the
+    output path only depends on the `outputHash*` and `name` attributes,
+    while all other attributes are ignored for the purpose of computing
+    the output path. (The `name` attribute is included because it is
+    part of the path.)
+    
+    As an example, here is the (simplified) Nix expression for
+    `fetchurl`:
+    
+        { stdenv, curl }: # The curl program is used for downloading.
+        
+        { url, sha256 }:
+        
+        stdenv.mkDerivation {
+          name = baseNameOf (toString url);
+          builder = ./builder.sh;
+          buildInputs = [ curl ];
+        
+          # This is a fixed-output derivation; the output must be a regular
+          # file with SHA256 hash sha256.
+          outputHashMode = "flat";
+          outputHashAlgo = "sha256";
+          outputHash = sha256;
+        
+          inherit url;
+        }
+    
+    The `outputHashAlgo` attribute specifies the hash algorithm used to
+    compute the hash. It can currently be `"sha1"`, `"sha256"` or
+    `"sha512"`.
+    
+    The `outputHashMode` attribute determines how the hash is computed.
+    It must be one of the following two values:
+    
+      - `"flat"`  
+        The output must be a non-executable regular file. If it isn’t,
+        the build fails. The hash is simply computed over the contents
+        of that file (so it’s equal to what Unix commands like
+        `sha256sum` or `sha1sum` produce).
+        
+        This is the default.
+    
+      - `"recursive"`  
+        The hash is computed over the NAR archive dump of the output
+        (i.e., the result of [`nix-store
+                                                                        --dump`](#refsec-nix-store-dump)). In this case, the output can
+        be anything, including a directory tree.
+    
+    The `outputHash` attribute, finally, must be a string containing the
+    hash in either hexadecimal or base-32 notation. (See the [`nix-hash`
+    command](#sec-nix-hash) for information about converting to and from
+    base-32 notation.)
+
+  - `passAsFile`  
+    A list of names of attributes that should be passed via files rather
+    than environment variables. For example, if you have
+    
+    ``` 
+    passAsFile = ["big"];
+    big = "a very long string";
+        
+    ```
+    
+    then when the builder runs, the environment variable bigPath will
+    contain the absolute path to a temporary file containing `a very
+    long
+                    string`. That is, for any attribute x listed in `passAsFile`, Nix
+    will pass an environment variable xPath holding the path of the file
+    containing the value of attribute x. This is useful when you need to
+    pass large strings to a builder, since most operating systems impose
+    a limit on the size of the environment (typically, a few hundred
+    kilobyte).
+
+  - `preferLocalBuild`  
+    If this attribute is set to `true` and [distributed building is
+    enabled](#chap-distributed-builds), then, if possible, the derivaton
+    will be built locally instead of forwarded to a remote machine. This
+    is appropriate for trivial builders where the cost of doing a
+    download or remote build would exceed the cost of building locally.
+
+  - `allowSubstitutes`  
+    If this attribute is set to `false`, then Nix will always build this
+    derivation; it will not try to substitute its outputs. This is
+    useful for very trivial derivations (such as `writeText` in Nixpkgs)
+    that are cheaper to build than to substitute from a binary cache.
+    
+    > **Note**
+    > 
+    > You need to have a builder configured which satisfies the
+    > derivation’s `system` attribute, since the derivation cannot be
+    > substituted. Thus it is usually a good idea to align `system` with
+    > `builtins.currentSystem` when setting `allowSubstitutes` to
+    > `false`. For most trivial derivations this should be the case.
diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md
new file mode 100644
index 000000000..9a373d94d
--- /dev/null
+++ b/doc/manual/src/expressions/arguments-variables.md
@@ -0,0 +1,72 @@
+# Arguments and Variables
+
+    ...
+    
+    rec { 
+    
+      hello = import ../applications/misc/hello/ex-1  { 
+        inherit fetchurl stdenv perl;
+      };
+    
+      perl = import ../development/interpreters/perl { 
+        inherit fetchurl stdenv;
+      };
+    
+      fetchurl = import ../build-support/fetchurl {
+        inherit stdenv; ...
+      };
+    
+      stdenv = ...;
+    
+    }
+
+The Nix expression in [???](#ex-hello-nix) is a function; it is missing
+some arguments that have to be filled in somewhere. In the Nix Packages
+collection this is done in the file `pkgs/top-level/all-packages.nix`,
+where all Nix expressions for packages are imported and called with the
+appropriate arguments. [example\_title](#ex-hello-composition) shows
+some fragments of `all-packages.nix`.
+
+  - This file defines a set of attributes, all of which are concrete
+    derivations (i.e., not functions). In fact, we define a *mutually
+    recursive* set of attributes. That is, the attributes can refer to
+    each other. This is precisely what we want since we want to “plug”
+    the various packages into each other.
+
+  - Here we *import* the Nix expression for GNU Hello. The import
+    operation just loads and returns the specified Nix expression. In
+    fact, we could just have put the contents of [???](#ex-hello-nix) in
+    `all-packages.nix` at this point. That would be completely
+    equivalent, but it would make the file rather bulky.
+    
+    Note that we refer to `../applications/misc/hello/ex-1`, not
+    `../applications/misc/hello/ex-1/default.nix`. When you try to
+    import a directory, Nix automatically appends `/default.nix` to the
+    file name.
+
+  - This is where the actual composition takes place. Here we *call* the
+    function imported from `../applications/misc/hello/ex-1` with a set
+    containing the things that the function expects, namely `fetchurl`,
+    `stdenv`, and `perl`. We use inherit again to use the attributes
+    defined in the surrounding scope (we could also have written
+    `fetchurl = fetchurl;`, etc.).
+    
+    The result of this function call is an actual derivation that can be
+    built by Nix (since when we fill in the arguments of the function,
+    what we get is its body, which is the call to `stdenv.mkDerivation`
+    in [???](#ex-hello-nix)).
+    
+    > **Note**
+    > 
+    > Nixpkgs has a convenience function `callPackage` that imports and
+    > calls a function, filling in any missing arguments by passing the
+    > corresponding attribute from the Nixpkgs set, like this:
+    > 
+    >     hello = callPackage ../applications/misc/hello/ex-1 { };
+    > 
+    > If necessary, you can set or override arguments:
+    > 
+    >     hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; };
+
+  - Likewise, we have to instantiate Perl, `fetchurl`, and the standard
+    environment.
diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md
new file mode 100644
index 000000000..256a5cd44
--- /dev/null
+++ b/doc/manual/src/expressions/build-script.md
@@ -0,0 +1,72 @@
+# Build Script
+
+    source $stdenv/setup 
+    
+    PATH=$perl/bin:$PATH 
+    
+    tar xvfz $src 
+    cd hello-*
+    ./configure --prefix=$out 
+    make 
+    make install
+
+[example\_title](#ex-hello-builder) shows the builder referenced from
+Hello's Nix expression (stored in
+`pkgs/applications/misc/hello/ex-1/builder.sh`). The builder can
+actually be made a lot shorter by using the *generic builder* functions
+provided by `stdenv`, but here we write out the build steps to elucidate
+what a builder does. It performs the following steps:
+
+  - When Nix runs a builder, it initially completely clears the
+    environment (except for the attributes declared in the derivation).
+    For instance, the PATH variable is empty\[1\]. This is done to
+    prevent undeclared inputs from being used in the build process. If
+    for example the PATH contained `/usr/bin`, then you might
+    accidentally use `/usr/bin/gcc`.
+    
+    So the first step is to set up the environment. This is done by
+    calling the `setup` script of the standard environment. The
+    environment variable stdenv points to the location of the standard
+    environment being used. (It wasn't specified explicitly as an
+    attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
+    automatically.)
+
+  - Since Hello needs Perl, we have to make sure that Perl is in the
+    PATH. The perl environment variable points to the location of the
+    Perl package (since it was passed in as an attribute to the
+    derivation), so `$perl/bin` is the directory containing the Perl
+    interpreter.
+
+  - Now we have to unpack the sources. The `src` attribute was bound to
+    the result of fetching the Hello source tarball from the network, so
+    the src environment variable points to the location in the Nix store
+    to which the tarball was downloaded. After unpacking, we `cd` to the
+    resulting source directory.
+    
+    The whole build is performed in a temporary directory created in
+    `/tmp`, by the way. This directory is removed after the builder
+    finishes, so there is no need to clean up the sources afterwards.
+    Also, the temporary directory is always newly created, so you don't
+    have to worry about files from previous builds interfering with the
+    current build.
+
+  - GNU Hello is a typical Autoconf-based package, so we first have to
+    run its `configure` script. In Nix every package is stored in a
+    separate location in the Nix store, for instance
+    `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix
+    computes this path by cryptographically hashing all attributes of
+    the derivation. The path is passed to the builder through the out
+    environment variable. So here we give `configure` the parameter
+    `--prefix=$out` to cause Hello to be installed in the expected
+    location.
+
+  - Finally we build Hello (`make`) and install it into the location
+    specified by out (`make install`).
+
+If you are wondering about the absence of error checking on the result
+of various commands called in the builder: this is because the shell
+script is evaluated with Bash's `-e` option, which causes the script to
+be aborted if any command fails without an error check.
+
+1.  Actually, it's initialised to `/path-not-set` to prevent Bash from
+    setting it to a default value.
diff --git a/doc/manual/src/expressions/builder-syntax.md b/doc/manual/src/expressions/builder-syntax.md
new file mode 100644
index 000000000..c92acf106
--- /dev/null
+++ b/doc/manual/src/expressions/builder-syntax.md
@@ -0,0 +1,72 @@
+# Builder Syntax
+
+    source $stdenv/setup 
+    
+    PATH=$perl/bin:$PATH 
+    
+    tar xvfz $src 
+    cd hello-*
+    ./configure --prefix=$out 
+    make 
+    make install
+
+[example\_title](#ex-hello-builder) shows the builder referenced from
+Hello's Nix expression (stored in
+`pkgs/applications/misc/hello/ex-1/builder.sh`). The builder can
+actually be made a lot shorter by using the *generic builder* functions
+provided by `stdenv`, but here we write out the build steps to elucidate
+what a builder does. It performs the following steps:
+
+  - When Nix runs a builder, it initially completely clears the
+    environment (except for the attributes declared in the derivation).
+    For instance, the PATH variable is empty\[1\]. This is done to
+    prevent undeclared inputs from being used in the build process. If
+    for example the PATH contained `/usr/bin`, then you might
+    accidentally use `/usr/bin/gcc`.
+    
+    So the first step is to set up the environment. This is done by
+    calling the `setup` script of the standard environment. The
+    environment variable stdenv points to the location of the standard
+    environment being used. (It wasn't specified explicitly as an
+    attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
+    automatically.)
+
+  - Since Hello needs Perl, we have to make sure that Perl is in the
+    PATH. The perl environment variable points to the location of the
+    Perl package (since it was passed in as an attribute to the
+    derivation), so `$perl/bin` is the directory containing the Perl
+    interpreter.
+
+  - Now we have to unpack the sources. The `src` attribute was bound to
+    the result of fetching the Hello source tarball from the network, so
+    the src environment variable points to the location in the Nix store
+    to which the tarball was downloaded. After unpacking, we `cd` to the
+    resulting source directory.
+    
+    The whole build is performed in a temporary directory created in
+    `/tmp`, by the way. This directory is removed after the builder
+    finishes, so there is no need to clean up the sources afterwards.
+    Also, the temporary directory is always newly created, so you don't
+    have to worry about files from previous builds interfering with the
+    current build.
+
+  - GNU Hello is a typical Autoconf-based package, so we first have to
+    run its `configure` script. In Nix every package is stored in a
+    separate location in the Nix store, for instance
+    `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix
+    computes this path by cryptographically hashing all attributes of
+    the derivation. The path is passed to the builder through the out
+    environment variable. So here we give `configure` the parameter
+    `--prefix=$out` to cause Hello to be installed in the expected
+    location.
+
+  - Finally we build Hello (`make`) and install it into the location
+    specified by out (`make install`).
+
+If you are wondering about the absence of error checking on the result
+of various commands called in the builder: this is because the shell
+script is evaluated with Bash's `-e` option, which causes the script to
+be aborted if any command fails without an error check.
+
+1.  Actually, it's initialised to `/path-not-set` to prevent Bash from
+    setting it to a default value.
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
new file mode 100644
index 000000000..a378fe8e4
--- /dev/null
+++ b/doc/manual/src/expressions/builtins.md
@@ -0,0 +1,839 @@
+# Built-in Functions
+
+This section lists the functions and constants built into the Nix
+expression evaluator. (The built-in function `derivation` is discussed
+above.) Some built-ins, such as `derivation`, are always in scope of
+every Nix expression; you can just access them right away. But to
+prevent polluting the namespace too much, most built-ins are not in
+scope. Instead, you can access them through the `builtins` built-in
+value, which is a set that contains all built-in functions and values.
+For instance, `derivation` is also available as `builtins.derivation`.
+
+  - `abort` s; `builtins.abort` s  
+    Abort Nix expression evaluation, print error message s.
+
+  - `builtins.add` e1 e2  
+    Return the sum of the numbers e1 and e2.
+
+  - `builtins.all` pred list  
+    Return `true` if the function pred returns `true` for all elements
+    of list, and `false` otherwise.
+
+  - `builtins.any` pred list  
+    Return `true` if the function pred returns `true` for at least one
+    element of list, and `false` otherwise.
+
+  - `builtins.attrNames` set  
+    Return the names of the attributes in the set set in an
+    alphabetically sorted list. For instance, `builtins.attrNames { y
+    = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`.
+
+  - `builtins.attrValues` set  
+    Return the values of the attributes in the set set in the order
+    corresponding to the sorted attribute names.
+
+  - `baseNameOf` s  
+    Return the *base name* of the string s, that is, everything
+    following the final slash in the string. This is similar to the GNU
+    `basename` command.
+
+  - `builtins.bitAnd` e1 e2  
+    Return the bitwise AND of the integers e1 and e2.
+
+  - `builtins.bitOr` e1 e2  
+    Return the bitwise OR of the integers e1 and e2.
+
+  - `builtins.bitXor` e1 e2  
+    Return the bitwise XOR of the integers e1 and e2.
+
+  - `builtins`  
+    The set `builtins` contains all the built-in functions and values.
+    You can use `builtins` to test for the availability of features in
+    the Nix installation, e.g.,
+    
+        if builtins ? getEnv then builtins.getEnv "PATH" else ""
+    
+    This allows a Nix expression to fall back gracefully on older Nix
+    installations that don’t have the desired built-in function.
+
+  - `builtins.compareVersions` s1 s2  
+    Compare two strings representing versions and return `-1` if version
+    s1 is older than version s2, `0` if they are the same, and `1` if s1
+    is newer than s2. The version comparison algorithm is the same as
+    the one used by [`nix-env
+                    -u`](#ssec-version-comparisons).
+
+  - `builtins.concatLists` lists  
+    Concatenate a list of lists into a single list.
+
+  - `builtins.concatStringsSep` separator list  
+    Concatenate a list of strings with a separator between each element,
+    e.g. `concatStringsSep "/"
+                    ["usr" "local" "bin"] == "usr/local/bin"`
+
+  - `builtins.currentSystem`  
+    The built-in value `currentSystem` evaluates to the Nix platform
+    identifier for the Nix installation on which the expression is being
+    evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
+
+  - `builtins.deepSeq` e1 e2  
+    This is like `seq
+                    e1
+                    e2`, except that e1 is evaluated *deeply*: if it’s a list or set,
+    its elements or attributes are also evaluated recursively.
+
+  - `derivation` attrs; `builtins.derivation` attrs  
+    `derivation` is described in [???](#ssec-derivation).
+
+  - `dirOf` s; `builtins.dirOf` s  
+    Return the directory part of the string s, that is, everything
+    before the final slash in the string. This is similar to the GNU
+    `dirname` command.
+
+  - `builtins.div` e1 e2  
+    Return the quotient of the numbers e1 and e2.
+
+  - `builtins.elem` x xs  
+    Return `true` if a value equal to x occurs in the list xs, and
+    `false` otherwise.
+
+  - `builtins.elemAt` xs n  
+    Return element n from the list xs. Elements are counted starting
+    from 0. A fatal error occurs if the index is out of bounds.
+
+  - `builtins.fetchurl` url  
+    Download the specified URL and return the path of the downloaded
+    file. This function is not available if [restricted evaluation
+    mode](#conf-restrict-eval) is enabled.
+
+  - `fetchTarball` url; `builtins.fetchTarball` url  
+    Download the specified URL, unpack it and return the path of the
+    unpacked tree. The file must be a tape archive (`.tar`) compressed
+    with `gzip`, `bzip2` or `xz`. The top-level path component of the
+    files in the tarball is removed, so it is best if the tarball
+    contains a single directory at top level. The typical use of the
+    function is to obtain external Nix expression dependencies, such as
+    a particular version of Nixpkgs, e.g.
+    
+        with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
+        
+        stdenv.mkDerivation { … }
+    
+    The fetched tarball is cached for a certain amount of time (1 hour
+    by default) in `~/.cache/nix/tarballs/`. You can change the cache
+    timeout either on the command line with `--option tarball-ttl number
+    of seconds` or in the Nix configuration file with this option: ` 
+    number of seconds to cache `.
+    
+    Note that when obtaining the hash with ` nix-prefetch-url
+                     ` the option `--unpack` is required.
+    
+    This function can also verify the contents against a hash. In that
+    case, the function takes a set instead of a URL. The set requires
+    the attribute `url` and the attribute `sha256`, e.g.
+    
+        with import (fetchTarball {
+          url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
+          sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
+        }) {};
+        
+        stdenv.mkDerivation { … }
+    
+    This function is not available if [restricted evaluation
+    mode](#conf-restrict-eval) is enabled.
+
+  - `builtins.fetchGit` args  
+    Fetch a path from git. args can be a URL, in which case the HEAD of
+    the repo at that URL is fetched. Otherwise, it can be an attribute
+    with the following attributes (all except `url` optional):
+    
+      - url  
+        The URL of the repo.
+    
+      - name  
+        The name of the directory the repo should be exported to in the
+        store. Defaults to the basename of the URL.
+    
+      - rev  
+        The git revision to fetch. Defaults to the tip of `ref`.
+    
+      - ref  
+        The git ref to look for the requested revision under. This is
+        often a branch or tag name. Defaults to `HEAD`.
+        
+        By default, the `ref` value is prefixed with `refs/heads/`. As
+        of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
+        with `refs/`.
+    
+      - submodules  
+        A Boolean parameter that specifies whether submodules should be
+        checked out. Defaults to `false`.
+    
+    <!-- end list -->
+    
+        builtins.fetchGit {
+          url = "git@github.com:my-secret/repository.git";
+          ref = "master";
+          rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+        }
+    
+        builtins.fetchGit {
+          url = "https://github.com/NixOS/nix.git";
+          ref = "refs/heads/0.5-release";
+        }
+    
+    If the revision you're looking for is in the default branch of the
+    git repository you don't strictly need to specify the branch name in
+    the `ref` attribute.
+    
+    However, if the revision you're looking for is in a future branch
+    for the non-default branch you will need to specify the the `ref`
+    attribute as well.
+    
+        builtins.fetchGit {
+          url = "https://github.com/nixos/nix.git";
+          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+          ref = "1.11-maintenance";
+        }
+    
+    > **Note**
+    > 
+    > It is nice to always specify the branch which a revision belongs
+    > to. Without the branch being specified, the fetcher might fail if
+    > the default branch changes. Additionally, it can be confusing to
+    > try a commit from a non-default branch and see the fetch fail. If
+    > the branch is specified the fault is much more obvious.
+    
+    If the revision you're looking for is in the default branch of the
+    git repository you may omit the `ref` attribute.
+    
+        builtins.fetchGit {
+          url = "https://github.com/nixos/nix.git";
+          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+        }
+    
+        builtins.fetchGit {
+          url = "https://github.com/nixos/nix.git";
+          ref = "refs/tags/1.9";
+        }
+    
+    `builtins.fetchGit` can behave impurely fetch the latest version of
+    a remote branch.
+    
+    > **Note**
+    > 
+    > Nix will refetch the branch in accordance to
+    > [???](#conf-tarball-ttl).
+    
+    > **Note**
+    > 
+    > This behavior is disabled in *Pure evaluation mode*.
+    
+        builtins.fetchGit {
+          url = "ssh://git@github.com/nixos/nix.git";
+          ref = "master";
+        }
+
+  - `builtins.filter` f xs  
+    Return a list consisting of the elements of xs for which the
+    function f returns `true`.
+
+  - `builtins.filterSource` e1 e2  
+    This function allows you to copy sources into the Nix store while
+    filtering certain files. For instance, suppose that you want to use
+    the directory `source-dir` as an input to a Nix expression, e.g.
+    
+        stdenv.mkDerivation {
+          ...
+          src = ./source-dir;
+        }
+    
+    However, if `source-dir` is a Subversion working copy, then all
+    those annoying `.svn` subdirectories will also be copied to the
+    store. Worse, the contents of those directories may change a lot,
+    causing lots of spurious rebuilds. With `filterSource` you can
+    filter out the `.svn` directories:
+    
+    ``` 
+      src = builtins.filterSource
+        (path: type: type != "directory" || baseNameOf path != ".svn")
+        ./source-dir;
+    ```
+    
+    Thus, the first argument e1 must be a predicate function that is
+    called for each regular file, directory or symlink in the source
+    tree e2. If the function returns `true`, the file is copied to the
+    Nix store, otherwise it is omitted. The function is called with two
+    arguments. The first is the full path of the file. The second is a
+    string that identifies the type of the file, which is either
+    `"regular"`, `"directory"`, `"symlink"` or `"unknown"` (for other
+    kinds of files such as device nodes or fifos — but note that those
+    cannot be copied to the Nix store, so if the predicate returns
+    `true` for them, the copy will fail). If you exclude a directory,
+    the entire corresponding subtree of e2 will be excluded.
+
+  - `builtins.foldl’` op nul list  
+    Reduce a list by applying a binary operator, from left to right,
+    e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op
+                    (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e.,
+    its arguments are evaluated first. For example, `foldl’ (x: y: x +
+    y) 0 [1 2 3]` evaluates to 6.
+
+  - `builtins.functionArgs` f  
+    Return a set containing the names of the formal arguments expected
+    by the function f. The value of each attribute is a Boolean denoting
+    whether the corresponding argument has a default value. For
+    instance, `functionArgs ({ x, y ? 123}: ...) = { x = false; y =
+    true; }`.
+    
+    "Formal argument" here refers to the attributes pattern-matched by
+    the function. Plain lambdas are not included, e.g. `functionArgs (x:
+    ...) = { }`.
+
+  - `builtins.fromJSON` e  
+    Convert a JSON string to a Nix value. For example,
+    
+        builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
+    
+    returns the value `{ x = [ 1 2 3 ]; y = null;
+                    }`.
+
+  - `builtins.genList` generator length  
+    Generate list of size length, with each element i equal to the value
+    returned by generator `i`. For example,
+    
+        builtins.genList (x: x * x) 5
+    
+    returns the list `[ 0 1 4 9 16 ]`.
+
+  - `builtins.getAttr` s set  
+    `getAttr` returns the attribute named s from set. Evaluation aborts
+    if the attribute doesn’t exist. This is a dynamic version of the `.`
+    operator, since s is an expression rather than an identifier.
+
+  - `builtins.getEnv` s  
+    `getEnv` returns the value of the environment variable s, or an
+    empty string if the variable doesn’t exist. This function should be
+    used with care, as it can introduce all sorts of nasty environment
+    dependencies in your Nix expression.
+    
+    `getEnv` is used in Nix Packages to locate the file
+    `~/.nixpkgs/config.nix`, which contains user-local settings for Nix
+    Packages. (That is, it does a `getEnv "HOME"` to locate the user’s
+    home directory.)
+
+  - `builtins.hasAttr` s set  
+    `hasAttr` returns `true` if set has an attribute named s, and
+    `false` otherwise. This is a dynamic version of the `?` operator,
+    since s is an expression rather than an identifier.
+
+  - `builtins.hashString` type s  
+    Return a base-16 representation of the cryptographic hash of string
+    s. The hash algorithm specified by type must be one of `"md5"`,
+    `"sha1"`, `"sha256"` or `"sha512"`.
+
+  - `builtins.hashFile` type p  
+    Return a base-16 representation of the cryptographic hash of the
+    file at path p. The hash algorithm specified by type must be one of
+    `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`.
+
+  - `builtins.head` list  
+    Return the first element of a list; abort evaluation if the argument
+    isn’t a list or is an empty list. You can test whether a list is
+    empty by comparing it with `[]`.
+
+  - `import` path; `builtins.import` path  
+    Load, parse and return the Nix expression in the file path. If path
+    is a directory, the file ` default.nix
+                     ` in that directory is loaded. Evaluation aborts if the file
+    doesn’t exist or contains an incorrect Nix expression. `import`
+    implements Nix’s module system: you can put any Nix expression (such
+    as a set or a function) in a separate file, and use it from Nix
+    expressions in other files.
+    
+    > **Note**
+    > 
+    > Unlike some languages, `import` is a regular function in Nix.
+    > Paths using the angle bracket syntax (e.g., `
+    >     >     >     >     > import` \<foo\>) are normal path values (see [???](#ssec-values)).
+    
+    A Nix expression loaded by `import` must not contain any *free
+    variables* (identifiers that are not defined in the Nix expression
+    itself and are not built-in). Therefore, it cannot refer to
+    variables that are in scope at the call site. For instance, if you
+    have a calling expression
+    
+        rec {
+          x = 123;
+          y = import ./foo.nix;
+        }
+    
+    then the following `foo.nix` will give an error:
+    
+        x + 456
+    
+    since `x` is not in scope in `foo.nix`. If you want `x` to be
+    available in `foo.nix`, you should pass it as a function argument:
+    
+        rec {
+          x = 123;
+          y = import ./foo.nix x;
+        }
+    
+    and
+    
+        x: x + 456
+    
+    (The function argument doesn’t have to be called `x` in `foo.nix`;
+    any name would work.)
+
+  - `builtins.intersectAttrs` e1 e2  
+    Return a set consisting of the attributes in the set e2 that also
+    exist in the set e1.
+
+  - `builtins.isAttrs` e  
+    Return `true` if e evaluates to a set, and `false` otherwise.
+
+  - `builtins.isList` e  
+    Return `true` if e evaluates to a list, and `false` otherwise.
+
+  - `builtins.isFunction` e  
+    Return `true` if e evaluates to a function, and `false` otherwise.
+
+  - `builtins.isString` e  
+    Return `true` if e evaluates to a string, and `false` otherwise.
+
+  - `builtins.isInt` e  
+    Return `true` if e evaluates to an int, and `false` otherwise.
+
+  - `builtins.isFloat` e  
+    Return `true` if e evaluates to a float, and `false` otherwise.
+
+  - `builtins.isBool` e  
+    Return `true` if e evaluates to a bool, and `false` otherwise.
+
+  - `builtins.isPath` e  
+    Return `true` if e evaluates to a path, and `false` otherwise.
+
+  - `isNull` e; `builtins.isNull` e  
+    Return `true` if e evaluates to `null`, and `false` otherwise.
+    
+    > **Warning**
+    > 
+    > This function is *deprecated*; just write `e == null` instead.
+
+  - `builtins.length` e  
+    Return the length of the list e.
+
+  - `builtins.lessThan` e1 e2  
+    Return `true` if the number e1 is less than the number e2, and
+    `false` otherwise. Evaluation aborts if either e1 or e2 does not
+    evaluate to a number.
+
+  - `builtins.listToAttrs` e  
+    Construct a set from a list specifying the names and values of each
+    attribute. Each element of the list should be a set consisting of a
+    string-valued attribute `name` specifying the name of the attribute,
+    and an attribute `value` specifying its value. Example:
+    
+        builtins.listToAttrs
+          [ { name = "foo"; value = 123; }
+            { name = "bar"; value = 456; }
+          ]
+    
+    evaluates to
+    
+        { foo = 123; bar = 456; }
+
+  - `map` f list; `builtins.map` f list  
+    Apply the function f to each element in the list list. For example,
+    
+        map (x: "foo" + x) [ "bar" "bla" "abc" ]
+    
+    evaluates to `[ "foobar" "foobla" "fooabc"
+                    ]`.
+
+  - `builtins.match` regex str  
+    Returns a list if the [extended POSIX regular
+    expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
+    regex matches str precisely, otherwise returns `null`. Each item in
+    the list is a regex group.
+    
+        builtins.match "ab" "abc"
+    
+    Evaluates to `null`.
+    
+        builtins.match "abc" "abc"
+    
+    Evaluates to `[ ]`.
+    
+        builtins.match "a(b)(c)" "abc"
+    
+    Evaluates to `[ "b" "c" ]`.
+    
+        builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
+    
+    Evaluates to `[ "foo" ]`.
+
+  - `builtins.mul` e1 e2  
+    Return the product of the numbers e1 and e2.
+
+  - `builtins.parseDrvName` s  
+    Split the string s into a package name and version. The package name
+    is everything up to but not including the first dash followed by a
+    digit, and the version is everything following that dash. The result
+    is returned in a set `{ name, version }`. Thus,
+    `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = "nix";
+    version = "0.12pre12876";
+                    }`.
+
+  - `builtins.path` args  
+    An enrichment of the built-in path type, based on the attributes
+    present in args. All are optional except `path`:
+    
+      - path  
+        The underlying path.
+    
+      - name  
+        The name of the path when added to the store. This can used to
+        reference paths that have nix-illegal characters in their names,
+        like `@`.
+    
+      - filter  
+        A function of the type expected by
+        [builtins.filterSource](#builtin-filterSource), with the same
+        semantics.
+    
+      - recursive  
+        When `false`, when `path` is added to the store it is with a
+        flat hash, rather than a hash of the NAR serialization of the
+        file. Thus, `path` must refer to a regular file, not a
+        directory. This allows similar behavior to `fetchurl`. Defaults
+        to `true`.
+    
+      - sha256  
+        When provided, this is the expected hash of the file at the
+        path. Evaluation will fail if the hash is incorrect, and
+        providing a hash allows `builtins.path` to be used even when the
+        `pure-eval` nix config option is on.
+
+  - `builtins.pathExists` path  
+    Return `true` if the path path exists at evaluation time, and
+    `false` otherwise.
+
+  - `builtins.placeholder` output  
+    Return a placeholder string for the specified output that will be
+    substituted by the corresponding output path at build time. Typical
+    outputs would be `"out"`, `"bin"` or `"dev"`.
+
+  - `builtins.readDir` path  
+    Return the contents of the directory path as a set mapping directory
+    entries to the corresponding file type. For instance, if directory
+    `A` contains a regular file `B` and another directory `C`, then
+    `builtins.readDir
+                    ./A` will return the set
+    
+        { B = "regular"; C = "directory"; }
+    
+    The possible values for the file type are `"regular"`,
+    `"directory"`, `"symlink"` and `"unknown"`.
+
+  - `builtins.readFile` path  
+    Return the contents of the file path as a string.
+
+  - `removeAttrs` set list; `builtins.removeAttrs` set list  
+    Remove the attributes listed in list from set. The attributes don’t
+    have to exist in set. For instance,
+    
+        removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]
+    
+    evaluates to `{ y = 2; }`.
+
+  - `builtins.replaceStrings` from to s  
+    Given string s, replace every occurrence of the strings in from with
+    the corresponding string in to. For example,
+    
+        builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
+    
+    evaluates to `"fabir"`.
+
+  - `builtins.seq` e1 e2  
+    Evaluate e1, then evaluate and return e2. This ensures that a
+    computation is strict in the value of e1.
+
+  - `builtins.sort` comparator list  
+    Return list in sorted order. It repeatedly calls the function
+    comparator with two elements. The comparator should return `true` if
+    the first element is less than the second, and `false` otherwise.
+    For example,
+    
+        builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
+    
+    produces the list `[ 42 77 147 249 483 526
+                    ]`.
+    
+    This is a stable sort: it preserves the relative order of elements
+    deemed equal by the comparator.
+
+  - `builtins.split` regex str  
+    Returns a list composed of non matched strings interleaved with the
+    lists of the [extended POSIX regular
+    expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
+    regex matches of str. Each item in the lists of matched sequences is
+    a regex group.
+    
+        builtins.split "(a)b" "abc"
+    
+    Evaluates to `[ "" [ "a" ] "c" ]`.
+    
+        builtins.split "([ac])" "abc"
+    
+    Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`.
+    
+        builtins.split "(a)|(c)" "abc"
+    
+    Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`.
+    
+        builtins.split "([[:upper:]]+)" "  FOO   "
+    
+    Evaluates to `[ " " [ "FOO" ] " " ]`.
+
+  - `builtins.splitVersion` s  
+    Split a string representing a version into its components, by the
+    same version splitting logic underlying the version comparison in
+    [`nix-env -u`](#ssec-version-comparisons).
+
+  - `builtins.stringLength` e  
+    Return the length of the string e. If e is not a string, evaluation
+    is aborted.
+
+  - `builtins.sub` e1 e2  
+    Return the difference between the numbers e1 and e2.
+
+  - `builtins.substring` start len s  
+    Return the substring of s from character position start (zero-based)
+    up to but not including start + len. If start is greater than the
+    length of the string, an empty string is returned, and if start +
+    len lies beyond the end of the string, only the substring up to the
+    end of the string is returned. start must be non-negative. For
+    example,
+    
+        builtins.substring 0 3 "nixos"
+    
+    evaluates to `"nix"`.
+
+  - `builtins.tail` list  
+    Return the second to last elements of a list; abort evaluation if
+    the argument isn’t a list or is an empty list.
+
+  - `throw` s; `builtins.throw` s  
+    Throw an error message s. This usually aborts Nix expression
+    evaluation, but in `nix-env -qa` and other commands that try to
+    evaluate a set of derivations to get information about those
+    derivations, a derivation that throws an error is silently skipped
+    (which is not the case for `abort`).
+
+  - `builtins.toFile` name s  
+    Store the string s in a file in the Nix store and return its path.
+    The file has suffix name. This file can be used as an input to
+    derivations. One application is to write builders “inline”. For
+    instance, the following Nix expression combines [???](#ex-hello-nix)
+    and [???](#ex-hello-builder) into one file:
+    
+        { stdenv, fetchurl, perl }:
+        
+        stdenv.mkDerivation {
+          name = "hello-2.1.1";
+        
+          builder = builtins.toFile "builder.sh" "
+            source $stdenv/setup
+        
+            PATH=$perl/bin:$PATH
+        
+            tar xvfz $src
+            cd hello-*
+            ./configure --prefix=$out
+            make
+            make install
+          ";
+        
+          src = fetchurl {
+            url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+            sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+          };
+          inherit perl;
+        }
+    
+    It is even possible for one file to refer to another, e.g.,
+    
+    ``` 
+      builder = let
+        configFile = builtins.toFile "foo.conf" "
+          # This is some dummy configuration file.
+          ...
+        ";
+      in builtins.toFile "builder.sh" "
+        source $stdenv/setup
+        ...
+        cp ${configFile} $out/etc/foo.conf
+      ";
+    ```
+    
+    Note that `${configFile}` is an antiquotation (see
+    [???](#ssec-values)), so the result of the expression `configFile`
+    (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
+    spliced into the resulting string.
+    
+    It is however *not* allowed to have files mutually referring to each
+    other, like so:
+    
+        let
+          foo = builtins.toFile "foo" "...${bar}...";
+          bar = builtins.toFile "bar" "...${foo}...";
+        in foo
+    
+    This is not allowed because it would cause a cyclic dependency in
+    the computation of the cryptographic hashes for `foo` and `bar`.
+    
+    It is also not possible to reference the result of a derivation. If
+    you are using Nixpkgs, the `writeTextFile` function is able to do
+    that.
+
+  - `builtins.toJSON` e  
+    Return a string containing a JSON representation of e. Strings,
+    integers, floats, booleans, nulls and lists are mapped to their JSON
+    equivalents. Sets (except derivations) are represented as objects.
+    Derivations are translated to a JSON string containing the
+    derivation’s output path. Paths are copied to the store and
+    represented as a JSON string of the resulting store path.
+
+  - `builtins.toPath` s  
+    DEPRECATED. Use `/. + "/path"` to convert a string into an absolute
+    path. For relative paths, use `./. + "/path"`.
+
+  - `toString` e; `builtins.toString` e  
+    Convert the expression e to a string. e can be:
+    
+      - A string (in which case the string is returned unmodified).
+    
+      - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`.
+    
+      - A set containing `{ __toString = self: ...; }`.
+    
+      - An integer.
+    
+      - A list, in which case the string representations of its elements
+        are joined with spaces.
+    
+      - A Boolean (`false` yields `""`, `true` yields `"1"`).
+    
+      - `null`, which yields the empty string.
+
+  - `builtins.toXML` e  
+    Return a string containing an XML representation of e. The main
+    application for `toXML` is to communicate information with the
+    builder in a more structured format than plain environment
+    variables.
+    
+    [example\_title](#ex-toxml) shows an example where this is the case.
+    The builder is supposed to generate the configuration file for a
+    [Jetty servlet container](http://jetty.mortbay.org/). A servlet
+    container contains a number of servlets (`*.war` files) each
+    exported under a specific URI prefix. So the servlet configuration
+    is a list of sets containing the `path` and `war` of the servlet
+    ([co\_title](#ex-toxml-co-servlets)). This kind of information is
+    difficult to communicate with the normal method of passing
+    information through an environment variable, which just concatenates
+    everything together into a string (which might just work in this
+    case, but wouldn’t work if fields are optional or contain lists
+    themselves). Instead the Nix expression is converted to an XML
+    representation with `toXML`, which is unambiguous and can easily be
+    processed with the appropriate tools. For instance, in the example
+    an XSLT stylesheet ([co\_title](#ex-toxml-co-stylesheet)) is applied
+    to it ([co\_title](#ex-toxml-co-apply)) to generate the XML
+    configuration file for the Jetty server. The XML representation
+    produced from [co\_title](#ex-toxml-co-servlets) by `toXML` is shown
+    in [example\_title](#ex-toxml-result).
+    
+    Note that [example\_title](#ex-toxml) uses the `toFile` built-in to
+    write the builder and the stylesheet “inline” in the Nix expression.
+    The path of the stylesheet is spliced into the builder at `xsltproc
+    ${stylesheet}
+                    ...`.
+    
+        { stdenv, fetchurl, libxslt, jira, uberwiki }:
+        
+        stdenv.mkDerivation (rec {
+          name = "web-server";
+        
+          buildInputs = [ libxslt ];
+        
+          builder = builtins.toFile "builder.sh" "
+            source $stdenv/setup
+            mkdir $out
+            echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml  
+          ";
+        
+          stylesheet = builtins.toFile "stylesheet.xsl"  
+           "<?xml version='1.0' encoding='UTF-8'?>
+            <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
+              <xsl:template match='/'>
+                <Configure>
+                  <xsl:for-each select='/expr/list/attrs'>
+                    <Call name='addWebApplication'>
+                      <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg>
+                      <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg>
+                    </Call>
+                  </xsl:for-each>
+                </Configure>
+              </xsl:template>
+            </xsl:stylesheet>
+          ";
+        
+          servlets = builtins.toXML [  
+            { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
+            { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
+          ];
+        })
+    
+        <?xml version='1.0' encoding='utf-8'?>
+        <expr>
+          <list>
+            <attrs>
+              <attr name="path">
+                <string value="/bugtracker" />
+              </attr>
+              <attr name="war">
+                <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" />
+              </attr>
+            </attrs>
+            <attrs>
+              <attr name="path">
+                <string value="/wiki" />
+              </attr>
+              <attr name="war">
+                <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" />
+              </attr>
+            </attrs>
+          </list>
+        </expr>
+
+  - `builtins.trace` e1 e2  
+    Evaluate e1 and print its abstract syntax representation on standard
+    error. Then return e2. This function is useful for debugging.
+
+  - `builtins.tryEval` e  
+    Try to shallowly evaluate e. Return a set containing the attributes
+    `success` (`true` if e evaluated successfully, `false` if an error
+    was thrown) and `value`, equalling e if successful and `false`
+    otherwise. Note that this doesn't evaluate e deeply, so ` let e = {
+    x = throw ""; }; in (builtins.tryEval e).success
+                     ` will be `true`. Using ` builtins.deepSeq
+                     ` one can get the expected result: `let e = { x = throw "";
+                    }; in (builtins.tryEval (builtins.deepSeq e e)).success` will be
+    `false`.
+
+  - `builtins.typeOf` e  
+    Return a string representing the type of the value e, namely
+    `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`,
+    `"list"`, `"lambda"` or `"float"`.
diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/expressions/derivations.md
new file mode 100644
index 000000000..7ffc6fabe
--- /dev/null
+++ b/doc/manual/src/expressions/derivations.md
@@ -0,0 +1,154 @@
+# Derivations
+
+The most important built-in function is `derivation`, which is used to
+describe a single derivation (a build action). It takes as input a set,
+the attributes of which specify the inputs of the build.
+
+  - There must be an attribute named `system` whose value must be a
+    string specifying a Nix platform identifier, such as `"i686-linux"`
+    or `"x86_64-darwin"`\[1\] The build can only be performed on a
+    machine and operating system matching the platform identifier. (Nix
+    can automatically forward builds for other platforms by forwarding
+    them to other machines; see [???](#chap-distributed-builds).)
+
+  - There must be an attribute named `name` whose value must be a
+    string. This is used as a symbolic name for the package by
+    `nix-env`, and it is appended to the output paths of the derivation.
+
+  - There must be an attribute named `builder` that identifies the
+    program that is executed to perform the build. It can be either a
+    derivation or a source (a local file reference, e.g.,
+    `./builder.sh`).
+
+  - Every attribute is passed as an environment variable to the builder.
+    Attribute values are translated to environment variables as follows:
+    
+      - Strings and numbers are just passed verbatim.
+    
+      - A *path* (e.g., `../foo/sources.tar`) causes the referenced file
+        to be copied to the store; its location in the store is put in
+        the environment variable. The idea is that all sources should
+        reside in the Nix store, since all inputs to a derivation should
+        reside in the Nix store.
+    
+      - A *derivation* causes that derivation to be built prior to the
+        present derivation; its default output path is put in the
+        environment variable.
+    
+      - Lists of the previous types are also allowed. They are simply
+        concatenated, separated by spaces.
+    
+      - `true` is passed as the string `1`, `false` and `null` are
+        passed as an empty string.
+
+  - The optional attribute `args` specifies command-line arguments to be
+    passed to the builder. It should be a list.
+
+  - The optional attribute `outputs` specifies a list of symbolic
+    outputs of the derivation. By default, a derivation produces a
+    single output path, denoted as `out`. However, derivations can
+    produce multiple output paths. This is useful because it allows
+    outputs to be downloaded or garbage-collected separately. For
+    instance, imagine a library package that provides a dynamic library,
+    header files, and documentation. A program that links against the
+    library doesn’t need the header files and documentation at runtime,
+    and it doesn’t need the documentation at build time. Thus, the
+    library package could specify:
+    
+        outputs = [ "lib" "headers" "doc" ];
+    
+    This will cause Nix to pass environment variables `lib`, `headers`
+    and `doc` to the builder containing the intended store paths of each
+    output. The builder would typically do something like
+    
+        ./configure --libdir=$lib/lib --includedir=$headers/include --docdir=$doc/share/doc
+    
+    for an Autoconf-style package. You can refer to each output of a
+    derivation by selecting it as an attribute, e.g.
+    
+        buildInputs = [ pkg.lib pkg.headers ];
+    
+    The first element of `outputs` determines the *default output*.
+    Thus, you could also write
+    
+        buildInputs = [ pkg pkg.headers ];
+    
+    since `pkg` is equivalent to `pkg.lib`.
+
+The function `mkDerivation` in the Nixpkgs standard environment is a
+wrapper around `derivation` that adds a default value for `system` and
+always uses Bash as the builder, to which the supplied builder is passed
+as a command-line argument. See the Nixpkgs manual for details.
+
+The builder is executed as follows:
+
+  - A temporary directory is created under the directory specified by
+    TMPDIR (default `/tmp`) where the build will take place. The current
+    directory is changed to this directory.
+
+  - The environment is cleared and set to the derivation attributes, as
+    specified above.
+
+  - In addition, the following variables are set:
+    
+      - NIX\_BUILD\_TOP contains the path of the temporary directory for
+        this build.
+    
+      - Also, TMPDIR, TEMPDIR, TMP, TEMP are set to point to the
+        temporary directory. This is to prevent the builder from
+        accidentally writing temporary files anywhere else. Doing so
+        might cause interference by other processes.
+    
+      - PATH is set to `/path-not-set` to prevent shells from
+        initialising it to their built-in default value.
+    
+      - HOME is set to `/homeless-shelter` to prevent programs from
+        using `/etc/passwd` or the like to find the user's home
+        directory, which could cause impurity. Usually, when HOME is
+        set, it is used as the location of the home directory, even if
+        it points to a non-existent path.
+    
+      - NIX\_STORE is set to the path of the top-level Nix store
+        directory (typically, `/nix/store`).
+    
+      - For each output declared in `outputs`, the corresponding
+        environment variable is set to point to the intended path in the
+        Nix store for that output. Each output path is a concatenation
+        of the cryptographic hash of all build inputs, the `name`
+        attribute and the output name. (The output name is omitted if
+        it’s `out`.)
+
+  - If an output path already exists, it is removed. Also, locks are
+    acquired to prevent multiple Nix instances from performing the same
+    build at the same time.
+
+  - A log of the combined standard output and error is written to
+    `/nix/var/log/nix`.
+
+  - The builder is executed with the arguments specified by the
+    attribute `args`. If it exits with exit code 0, it is considered to
+    have succeeded.
+
+  - The temporary directory is removed (unless the `-K` option was
+    specified).
+
+  - If the build was successful, Nix scans each output path for
+    references to input paths by looking for the hash parts of the input
+    paths. Since these are potential runtime dependencies, Nix registers
+    them as dependencies of the output paths.
+
+  - After the build, Nix sets the last-modified timestamp on all files
+    in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to
+    the default group, and sets the mode of the file to 0444 or 0555
+    (i.e., read-only, with execute permission enabled if the file was
+    originally executable). Note that possible `setuid` and `setgid`
+    bits are cleared. Setuid and setgid programs are not currently
+    supported by Nix. This is because the Nix archives used in
+    deployment have no concept of ownership information, and because it
+    makes the build result dependent on the user performing the build.
+
+<!-- end list -->
+
+1.  To figure out your platform identifier, look at the line “Checking
+    for the canonical Nix system name” in the output of Nix's
+    `configure` script.
diff --git a/doc/manual/src/expressions/expression-language.md b/doc/manual/src/expressions/expression-language.md
new file mode 100644
index 000000000..267fcb983
--- /dev/null
+++ b/doc/manual/src/expressions/expression-language.md
@@ -0,0 +1,12 @@
+# Nix Expression Language
+
+The Nix expression language is a pure, lazy, functional language. Purity
+means that operations in the language don't have side-effects (for
+instance, there is no variable assignment). Laziness means that
+arguments to functions are evaluated only when they are needed.
+Functional means that functions are “normal” values that can be passed
+around and manipulated in interesting ways. The language is not a
+full-featured, general purpose language. Its main job is to describe
+packages, compositions of packages, and the variability within packages.
+
+This section presents the various features of the language.
diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md
new file mode 100644
index 000000000..935484c33
--- /dev/null
+++ b/doc/manual/src/expressions/expression-syntax.md
@@ -0,0 +1,91 @@
+# Expression Syntax
+
+    { stdenv, fetchurl, perl }: 
+    
+    stdenv.mkDerivation { 
+      name = "hello-2.1.1"; 
+      builder = ./builder.sh; 
+      src = fetchurl { 
+        url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+        sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+      };
+      inherit perl; 
+    }
+
+[example\_title](#ex-hello-nix) shows a Nix expression for GNU Hello.
+It's actually already in the Nix Packages collection in
+`pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to
+place each package in a separate directory and call the single Nix
+expression in that directory `default.nix`. The file has the following
+elements (referenced from the figure by number):
+
+  - This states that the expression is a *function* that expects to be
+    called with three arguments: `stdenv`, `fetchurl`, and `perl`. They
+    are needed to build Hello, but we don't know how to build them here;
+    that's why they are function arguments. `stdenv` is a package that
+    is used by almost all Nix Packages packages; it provides a
+    “standard” environment consisting of the things you would expect
+    in a basic Unix environment: a C/C++ compiler (GCC, to be precise),
+    the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`,
+    etc. `fetchurl` is a function that downloads files. `perl` is the
+    Perl interpreter.
+    
+    Nix functions generally have the form `{ x, y, ...,
+                    z }: e` where `x`, `y`, etc. are the names of the expected
+    arguments, and where e is the body of the function. So here, the
+    entire remainder of the file is the body of the function; when given
+    the required arguments, the body should describe how to build an
+    instance of the Hello package.
+
+  - So we have to build a package. Building something from other stuff
+    is called a *derivation* in Nix (as opposed to sources, which are
+    built by humans instead of computers). We perform a derivation by
+    calling `stdenv.mkDerivation`. `mkDerivation` is a function provided
+    by `stdenv` that builds a package from a set of *attributes*. A set
+    is just a list of key/value pairs where each key is a string and
+    each value is an arbitrary Nix expression. They take the general
+    form `{
+                    name1 =
+                    expr1; ...
+                    nameN =
+                    exprN; }`.
+
+  - The attribute `name` specifies the symbolic name and version of the
+    package. Nix doesn't really care about these things, but they are
+    used by for instance `nix-env
+                    -q` to show a “human-readable” name for packages. This attribute is
+    required by `mkDerivation`.
+
+  - The attribute `builder` specifies the builder. This attribute can
+    sometimes be omitted, in which case `mkDerivation` will fill in a
+    default builder (which does a `configure; make; make install`, in
+    essence). Hello is sufficiently simple that the default builder
+    would suffice, but in this case, we will show an actual builder for
+    educational purposes. The value `./builder.sh` refers to the shell
+    script shown in [???](#ex-hello-builder), discussed below.
+
+  - The builder has to know what the sources of the package are. Here,
+    the attribute `src` is bound to the result of a call to the
+    `fetchurl` function. Given a URL and a SHA-256 hash of the expected
+    contents of the file at that URL, this function builds a derivation
+    that downloads the file and checks its hash. So the sources are a
+    dependency that like all other dependencies is built before Hello
+    itself is built.
+    
+    Instead of `src` any other name could have been used, and in fact
+    there can be any number of sources (bound to different attributes).
+    However, `src` is customary, and it's also expected by the default
+    builder (which we don't use in this example).
+
+  - Since the derivation requires Perl, we have to pass the value of the
+    `perl` function argument to the builder. All attributes in the set
+    are actually passed as environment variables to the builder, so
+    declaring an attribute
+    
+        perl = perl;
+    
+    will do the trick: it binds an attribute `perl` to the function
+    argument which also happens to be called `perl`. However, it looks a
+    bit silly, so there is a shorter syntax. The `inherit` keyword
+    causes the specified attributes to be bound to whatever variables
+    with the same name happen to be in scope.
diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md
new file mode 100644
index 000000000..cbc484199
--- /dev/null
+++ b/doc/manual/src/expressions/generic-builder.md
@@ -0,0 +1,61 @@
+# Generic Builder Syntax
+
+Recall from [???](#ex-hello-builder) that the builder looked something
+like this:
+
+    PATH=$perl/bin:$PATH
+    tar xvfz $src
+    cd hello-*
+    ./configure --prefix=$out
+    make
+    make install
+
+The builders for almost all Unix packages look like this — set up some
+environment variables, unpack the sources, configure, build, and
+install. For this reason the standard environment provides some Bash
+functions that automate the build process. A builder using the generic
+build facilities in shown in [example\_title](#ex-hello-builder2).
+
+    buildInputs="$perl" 
+    
+    source $stdenv/setup 
+    
+    genericBuild 
+
+  - The buildInputs variable tells `setup` to use the indicated packages
+    as “inputs”. This means that if a package provides a `bin`
+    subdirectory, it's added to PATH; if it has a `include`
+    subdirectory, it's added to GCC's header search path; and so
+    on.\[1\]
+
+  - The function `genericBuild` is defined in the file `$stdenv/setup`.
+
+  - The final step calls the shell function `genericBuild`, which
+    performs the steps that were done explicitly in
+    [???](#ex-hello-builder). The generic builder is smart enough to
+    figure out whether to unpack the sources using `gzip`, `bzip2`, etc.
+    It can be customised in many ways; see the Nixpkgs manual for
+    details.
+
+Discerning readers will note that the buildInputs could just as well
+have been set in the Nix expression, like this:
+
+``` 
+  buildInputs = [ perl ];
+```
+
+The `perl` attribute can then be removed, and the builder becomes even
+shorter:
+
+    source $stdenv/setup
+    genericBuild
+
+In fact, `mkDerivation` provides a default builder that looks exactly
+like that, so it is actually possible to omit the builder for Hello
+entirely.
+
+1.  How does it work? `setup` tries to source the file
+    `pkg/nix-support/setup-hook` of all dependencies. These “setup
+    hooks” can then set up whatever environment variables they want;
+    for instance, the setup hook for Perl sets the PERL5LIB environment
+    variable to contain the `lib/site_perl` directories of all inputs.
diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/expressions/language-constructs.md
new file mode 100644
index 000000000..a773c239b
--- /dev/null
+++ b/doc/manual/src/expressions/language-constructs.md
@@ -0,0 +1,291 @@
+# Language Constructs
+
+Recursive sets are just normal sets, but the attributes can refer to
+each other. For example,
+
+    rec {
+      x = y;
+      y = 123;
+    }.x
+
+evaluates to `123`. Note that without `rec` the binding `x = y;` would
+refer to the variable `y` in the surrounding scope, if one exists, and
+would be invalid if no such variable exists. That is, in a normal
+(non-recursive) set, attributes are not added to the lexical scope; in a
+recursive set, they are.
+
+Recursive sets of course introduce the danger of infinite recursion. For
+example,
+
+    rec {
+      x = y;
+      y = x;
+    }.x
+
+does not terminate\[1\].
+
+A let-expression allows you to define local variables for an expression.
+For instance,
+
+    let
+      x = "foo";
+      y = "bar";
+    in x + y
+
+evaluates to `"foobar"`.
+
+When defining a set or in a let-expression it is often convenient to
+copy variables from the surrounding lexical scope (e.g., when you want
+to propagate attributes). This can be shortened using the `inherit`
+keyword. For instance,
+
+    let x = 123; in
+    { inherit x;
+      y = 456;
+    }
+
+is equivalent to
+
+    let x = 123; in
+    { x = x;
+      y = 456;
+    }
+
+and both evaluate to `{ x = 123; y = 456; }`. (Note that this works
+because `x` is added to the lexical scope by the `let` construct.) It is
+also possible to inherit attributes from another set. For instance, in
+this fragment from `all-packages.nix`,
+
+``` 
+  graphviz = (import ../tools/graphics/graphviz) {
+    inherit fetchurl stdenv libpng libjpeg expat x11 yacc;
+    inherit (xlibs) libXaw;
+  };
+
+  xlibs = {
+    libX11 = ...;
+    libXaw = ...;
+    ...
+  }
+
+  libpng = ...;
+  libjpg = ...;
+  ...
+```
+
+the set used in the function call to the function defined in
+`../tools/graphics/graphviz` inherits a number of variables from the
+surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw`
+(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set.
+
+Summarizing the fragment
+
+    ...
+    inherit x y z;
+    inherit (src-set) a b c;
+    ...
+
+is equivalent to
+
+    ...
+    x = x; y = y; z = z;
+    a = src-set.a; b = src-set.b; c = src-set.c;
+    ...
+
+when used while defining local variables in a let-expression or while
+defining a set.
+
+Functions have the following form:
+
+    pattern: body
+
+The pattern specifies what the argument of the function must look like,
+and binds variables in the body to (parts of) the argument. There are
+three kinds of patterns:
+
+  - If a pattern is a single identifier, then the function matches any
+    argument. Example:
+    
+        let negate = x: !x;
+            concat = x: y: x + y;
+        in if negate true then concat "foo" "bar" else ""
+    
+    Note that `concat` is a function that takes one argument and returns
+    a function that takes another argument. This allows partial
+    parameterisation (i.e., only filling some of the arguments of a
+    function); e.g.,
+    
+        map (concat "foo") [ "bar" "bla" "abc" ]
+    
+    evaluates to `[ "foobar" "foobla"
+            "fooabc" ]`.
+
+  - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a
+    set containing the listed attributes, and binds the values of those
+    attributes to variables in the function body. For example, the
+    function
+    
+        { x, y, z }: z + y + x
+    
+    can only be called with a set containing exactly the attributes `x`,
+    `y` and `z`. No other attributes are allowed. If you want to allow
+    additional arguments, you can use an ellipsis (`...`):
+    
+        { x, y, z, ... }: z + y + x
+    
+    This works on any set that contains at least the three named
+    attributes.
+    
+    It is possible to provide *default values* for attributes, in which
+    case they are allowed to be missing. A default value is specified by
+    writing `name ?
+            e`, where e is an arbitrary expression. For example,
+    
+        { x, y ? "foo", z ? "bar" }: z + y + x
+    
+    specifies a function that only requires an attribute named `x`, but
+    optionally accepts `y` and `z`.
+
+  - An `@`-pattern provides a means of referring to the whole value
+    being matched:
+    
+    ``` 
+     args@{ x, y, z, ... }: z + y + x + args.a
+    ```
+    
+    but can also be written as:
+    
+    ``` 
+     { x, y, z, ... } @ args: z + y + x + args.a
+    ```
+    
+    Here `args` is bound to the entire argument, which is further
+    matched against the pattern `{ x, y, z,
+            ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as
+    you can access attribute names as `a`, using `args.a`, which was
+    given as an additional attribute to the function.
+    
+    > **Warning**
+    > 
+    > The `args@` expression is bound to the argument passed to the
+    > function which means that attributes with defaults that aren't
+    > explicitly specified in the function call won't cause an
+    > evaluation error, but won't exist in `args`.
+    > 
+    > For instance
+    > 
+    >     let
+    >       function = args@{ a ? 23, ... }: args;
+    >     in
+    >      function {}
+    > 
+    > will evaluate to an empty attribute set.
+
+Note that functions do not have names. If you want to give them a name,
+you can bind them to an attribute, e.g.,
+
+    let concat = { x, y }: x + y;
+    in concat { x = "foo"; y = "bar"; }
+
+Conditionals look like this:
+
+    if e1 then e2 else e3
+
+where e1 is an expression that should evaluate to a Boolean value
+(`true` or `false`).
+
+Assertions are generally used to check that certain requirements on or
+between features and dependencies hold. They look like this:
+
+    assert e1; e2
+
+where e1 is an expression that should evaluate to a Boolean value. If it
+evaluates to `true`, e2 is returned; otherwise expression evaluation is
+aborted and a backtrace is printed.
+
+    { localServer ? false
+    , httpServer ? false
+    , sslSupport ? false
+    , pythonBindings ? false
+    , javaSwigBindings ? false
+    , javahlBindings ? false
+    , stdenv, fetchurl
+    , openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null
+    }:
+    
+    assert localServer -> db4 != null; 
+    assert httpServer -> httpd != null && httpd.expat == expat; 
+    assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); 
+    assert pythonBindings -> swig != null && swig.pythonSupport;
+    assert javaSwigBindings -> swig != null && swig.javaSupport;
+    assert javahlBindings -> j2sdk != null;
+    
+    stdenv.mkDerivation {
+      name = "subversion-1.1.1";
+      ...
+      openssl = if sslSupport then openssl else null; 
+      ...
+    }
+
+[example\_title](#ex-subversion-nix) show how assertions are used in the
+Nix expression for Subversion.
+
+  - This assertion states that if Subversion is to have support for
+    local repositories, then Berkeley DB is needed. So if the Subversion
+    function is called with the `localServer` argument set to `true` but
+    the `db4` argument set to `null`, then the evaluation fails.
+
+  - This is a more subtle condition: if Subversion is built with Apache
+    (`httpServer`) support, then the Expat library (an XML library) used
+    by Subversion should be same as the one used by Apache. This is
+    because in this configuration Subversion code ends up being linked
+    with Apache code, and if the Expat libraries do not match, a build-
+    or runtime link error or incompatibility might occur.
+
+  - This assertion says that in order for Subversion to have SSL support
+    (so that it can access `https` URLs), an OpenSSL library must be
+    passed. Additionally, it says that *if* Apache support is enabled,
+    then Apache's OpenSSL should match Subversion's. (Note that if
+    Apache support is not enabled, we don't care about Apache's
+    OpenSSL.)
+
+  - The conditional here is not really related to assertions, but is
+    worth pointing out: it ensures that if SSL support is disabled, then
+    the Subversion derivation is not dependent on OpenSSL, even if a
+    non-`null` value was passed. This prevents an unnecessary rebuild of
+    Subversion if OpenSSL changes.
+
+A *with-expression*,
+
+    with e1; e2
+
+introduces the set e1 into the lexical scope of the expression e2. For
+instance,
+
+    let as = { x = "foo"; y = "bar"; };
+    in with as; x + y
+
+evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes
+of `as` to the lexical scope in the expression `x + y`. The most common
+use of `with` is in conjunction with the `import` function. E.g.,
+
+    with (import ./definitions.nix); ...
+
+makes all attributes defined in the file `definitions.nix` available as
+if they were defined locally in a `let`-expression.
+
+The bindings introduced by `with` do not shadow bindings introduced by
+other means, e.g.
+
+    let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...
+
+establishes the same scope as
+
+    let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...
+
+Comments can be single-line, started with a `#` character, or
+inline/multi-line, enclosed within `/*
+... */`.
+
+1.  Actually, Nix detects infinite recursion in this case and aborts
+    (“infinite recursion encountered”).
diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md
new file mode 100644
index 000000000..4fa2eca37
--- /dev/null
+++ b/doc/manual/src/expressions/language-operators.md
@@ -0,0 +1,32 @@
+# Operators
+
+[table\_title](#table-operators) lists the operators in the Nix
+expression language, in order of precedence (from strongest to weakest
+binding).
+
+| Name                     | Syntax                        | Associativity | Description                                                                                                                                                                                                             | Precedence |
+| ------------------------ | ----------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
+| Select                   | e `.` attrpath \[ `or` def \] | none          | Select attribute denoted by the attribute path attrpath from set e. (An attribute path is a dot-separated list of attribute names.) If the attribute doesn’t exist, return def if provided, otherwise abort evaluation. | 1          |
+| Application              | e1 e2                         | left          | Call function e1 with argument e2.                                                                                                                                                                                      | 2          |
+| Arithmetic Negation      | `-` e                         | none          | Arithmetic negation.                                                                                                                                                                                                    | 3          |
+| Has Attribute            | e `?` attrpath                | none          | Test whether set e contains the attribute denoted by attrpath; return `true` or `false`.                                                                                                                                | 4          |
+| List Concatenation       | e1 `++` e2                    | right         | List concatenation.                                                                                                                                                                                                     | 5          |
+| Multiplication           | e1 `*` e2,                    | left          | Arithmetic multiplication.                                                                                                                                                                                              | 6          |
+| Division                 | e1 `/` e2                     | left          | Arithmetic division.                                                                                                                                                                                                    | 6          |
+| Addition                 | e1 `+` e2                     | left          | Arithmetic addition.                                                                                                                                                                                                    | 7          |
+| Subtraction              | e1 `-` e2                     | left          | Arithmetic subtraction.                                                                                                                                                                                                 | 7          |
+| String Concatenation     | string1 `+` string2           | left          | String concatenation.                                                                                                                                                                                                   | 7          |
+| Not                      | `!` e                         | none          | Boolean negation.                                                                                                                                                                                                       | 8          |
+| Update                   | e1 `//` e2                    | right         | Return a set consisting of the attributes in e1 and e2 (with the latter taking precedence over the former in case of equally named attributes).                                                                         | 9          |
+| Less Than                | e1 `<` e2,                    | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
+| Less Than or Equal To    | e1 `<=` e2                    | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
+| Greater Than             | e1 `>` e2                     | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
+| Greater Than or Equal To | e1 `>=` e2                    | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
+| Equality                 | e1 `==` e2                    | none          | Equality.                                                                                                                                                                                                               | 11         |
+| Inequality               | e1 `!=` e2                    | none          | Inequality.                                                                                                                                                                                                             | 11         |
+| Logical AND              | e1 `&&` e2                    | left          | Logical AND.                                                                                                                                                                                                            | 12         |
+| Logical OR               | e1 `\|\|` e2                  | left          | Logical OR.                                                                                                                                                                                                             | 13         |
+| Logical Implication      | e1 `->` e2                    | none          | Logical implication (equivalent to `!e1 \|\|
+        e2`).                                                                                                                                                              | 14         |
+
+Operators
diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md
new file mode 100644
index 000000000..98ad97c97
--- /dev/null
+++ b/doc/manual/src/expressions/language-values.md
@@ -0,0 +1,209 @@
+# Values
+
+Nix has the following basic data types:
+
+  - *Strings* can be written in three ways.
+    
+    The most common way is to enclose the string between double quotes,
+    e.g., `"foo bar"`. Strings can span multiple lines. The special
+    characters `"` and `\` and the character sequence `${` must be
+    escaped by prefixing them with a backslash (`\`). Newlines, carriage
+    returns and tabs can be written as `\n`, `\r` and `\t`,
+    respectively.
+    
+    You can include the result of an expression into a string by
+    enclosing it in `${...}`, a feature known as *antiquotation*. The
+    enclosed expression must evaluate to something that can be coerced
+    into a string (meaning that it must be a string, a path, or a
+    derivation). For instance, rather than writing
+    
+        "--with-freetype2-library=" + freetype + "/lib"
+    
+    (where `freetype` is a derivation), you can instead write the more
+    natural
+    
+        "--with-freetype2-library=${freetype}/lib"
+    
+    The latter is automatically translated to the former. A more
+    complicated example (from the Nix expression for
+    [Qt](http://www.trolltech.com/products/qt)):
+    
+        configureFlags = "
+          -system-zlib -system-libpng -system-libjpeg
+          ${if openglSupport then "-dlopen-opengl
+            -L${mesa}/lib -I${mesa}/include
+            -L${libXmu}/lib -I${libXmu}/include" else ""}
+          ${if threadSupport then "-thread" else "-no-thread"}
+        ";
+    
+    Note that Nix expressions and strings can be arbitrarily nested; in
+    this case the outer string contains various antiquotations that
+    themselves contain strings (e.g., `"-thread"`), some of which in
+    turn contain expressions (e.g., `${mesa}`).
+    
+    The second way to write string literals is as an *indented string*,
+    which is enclosed between pairs of *double single-quotes*, like so:
+    
+        ''
+          This is the first line.
+          This is the second line.
+            This is the third line.
+        ''
+    
+    This kind of string literal intelligently strips indentation from
+    the start of each line. To be precise, it strips from each line a
+    number of spaces equal to the minimal indentation of the string as a
+    whole (disregarding the indentation of empty lines). For instance,
+    the first and second line are indented two space, while the third
+    line is indented four spaces. Thus, two spaces are stripped from
+    each line, so the resulting string is
+    
+        "This is the first line.\nThis is the second line.\n  This is the third line.\n"
+    
+    Note that the whitespace and newline following the opening `''` is
+    ignored if there is no non-whitespace text on the initial line.
+    
+    Antiquotation (`${expr}`) is supported in indented strings.
+    
+    Since `${` and `''` have special meaning in indented strings, you
+    need a way to quote them. `$` can be escaped by prefixing it with
+    `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped
+    by prefixing it with `'`, i.e., `'''`. `$` removes any special
+    meaning from the following `$`. Linefeed, carriage-return and tab
+    characters can be written as `''\n`, `''\r`, `''\t`, and `''\`
+    escapes any other character.
+    
+    Indented strings are primarily useful in that they allow multi-line
+    string literals to follow the indentation of the enclosing Nix
+    expression, and that less escaping is typically necessary for
+    strings representing languages such as shell scripts and
+    configuration files because `''` is much less common than `"`.
+    Example:
+    
+        stdenv.mkDerivation {
+          ...
+          postInstall =
+            ''
+              mkdir $out/bin $out/etc
+              cp foo $out/bin
+              echo "Hello World" > $out/etc/foo.conf
+              ${if enableBar then "cp bar $out/bin" else ""}
+            '';
+          ...
+        }
+    
+    Finally, as a convenience, *URIs* as defined in appendix B of
+    [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as
+    is*, without quotes. For instance, the string
+    `"http://example.org/foo.tar.bz2"` can also be written as
+    `http://example.org/foo.tar.bz2`.
+
+  - Numbers, which can be *integers* (like `123`) or *floating point*
+    (like `123.43` or `.27e13`).
+    
+    Numbers are type-compatible: pure integer operations will always
+    return integers, whereas any operation involving at least one
+    floating point number will have a floating point number as a result.
+
+  - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at
+    least one slash to be recognised as such; for instance, `builder.sh`
+    is not a path\[1\]. If the file name is relative, i.e., if it does
+    not begin with a slash, it is made absolute at parse time relative
+    to the directory of the Nix expression that contained it. For
+    instance, if a Nix expression in `/foo/bar/bla.nix` refers to
+    `../xyzzy/fnord.nix`, the absolute path is `/foo/xyzzy/fnord.nix`.
+    
+    If the first component of a path is a `~`, it is interpreted as if
+    the rest of the path were relative to the user's home directory.
+    e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user
+    whose home directory is `/home/edolstra`.
+    
+    Paths can also be specified between angle brackets, e.g.
+    `<nixpkgs>`. This means that the directories listed in the
+    environment variable NIX\_PATH will be searched for the given file
+    or directory name.
+
+  - *Booleans* with values `true` and `false`.
+
+  - The null value, denoted as `null`.
+
+Lists are formed by enclosing a whitespace-separated list of values
+between square brackets. For example,
+
+    [ 123 ./foo.nix "abc" (f { x = y; }) ]
+
+defines a list of four elements, the last being the result of a call to
+the function `f`. Note that function calls have to be enclosed in
+parentheses. If they had been omitted, e.g.,
+
+    [ 123 ./foo.nix "abc" f { x = y; } ]
+
+the result would be a list of five elements, the fourth one being a
+function and the fifth being a set.
+
+Note that lists are only lazy in values, and they are strict in length.
+
+Sets are really the core of the language, since ultimately the Nix
+language is all about creating derivations, which are really just sets
+of attributes to be passed to build scripts.
+
+Sets are just a list of name/value pairs (called *attributes*) enclosed
+in curly brackets, where each value is an arbitrary expression
+terminated by a semicolon. For example:
+
+    { x = 123;
+      text = "Hello";
+      y = f { bla = 456; };
+    }
+
+This defines a set with attributes named `x`, `text`, `y`. The order of
+the attributes is irrelevant. An attribute name may only occur once.
+
+Attributes can be selected from a set using the `.` operator. For
+instance,
+
+    { a = "Foo"; b = "Bar"; }.a
+
+evaluates to `"Foo"`. It is possible to provide a default value in an
+attribute selection using the `or` keyword. For example,
+
+    { a = "Foo"; b = "Bar"; }.c or "Xyzzy"
+
+will evaluate to `"Xyzzy"` because there is no `c` attribute in the set.
+
+You can use arbitrary double-quoted strings as attribute names:
+
+    { "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}"
+
+This will evaluate to `123` (Assuming `bar` is antiquotable). In the
+case where an attribute name is just a single antiquotation, the quotes
+can be dropped:
+
+    { foo = 123; }.${bar} or 456 
+
+This will evaluate to `123` if `bar` evaluates to `"foo"` when coerced
+to a string and `456` otherwise (again assuming `bar` is antiquotable).
+
+In the special case where an attribute name inside of a set declaration
+evaluates to `null` (which is normally an error, as `null` is not
+antiquotable), that attribute is simply not added to the set:
+
+    { ${if foo then "bar" else null} = true; }
+
+This will evaluate to `{}` if `foo` evaluates to `false`.
+
+A set that has a `__functor` attribute whose value is callable (i.e. is
+itself a function or a set with a `__functor` attribute whose value is
+callable) can be applied as if it were a function, with the set itself
+passed in first , e.g.,
+
+    let add = { __functor = self: x: x + self.x; };
+        inc = add // { x = 1; };
+    in inc 1
+
+evaluates to `2`. This can be used to attach metadata to a function
+without the caller needing to treat it specially, or to implement a form
+of object-oriented programming, for example.
+
+1.  It's parsed as an expression that selects the attribute `sh` from
+    the variable `builder`.
diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md
new file mode 100644
index 000000000..bc064c733
--- /dev/null
+++ b/doc/manual/src/expressions/simple-building-testing.md
@@ -0,0 +1,57 @@
+# Building and Testing
+
+You can now try to build Hello. Of course, you could do `nix-env -i
+hello`, but you may not want to install a possibly broken package just
+yet. The best way to test the package is by using the command
+`nix-build`, which builds a Nix expression and creates a symlink named
+`result` in the current directory:
+
+    $ nix-build -A hello
+    building path `/nix/store/632d2b22514d...-hello-2.1.1'
+    hello-2.1.1/
+    hello-2.1.1/intl/
+    hello-2.1.1/intl/ChangeLog
+    ...
+    
+    $ ls -l result
+    lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1
+    
+    $ ./result/bin/hello
+    Hello, world!
+
+The [`-A`](#opt-attr) option selects the `hello` attribute. This is
+faster than using the symbolic package name specified by the `name`
+attribute (which also happens to be `hello`) and is unambiguous (there
+can be multiple packages with the symbolic name `hello`, but there can
+be only one attribute in a set named `hello`).
+
+`nix-build` registers the `./result` symlink as a garbage collection
+root, so unless and until you delete the `./result` symlink, the output
+of the build will be safely kept on your system. You can use
+`nix-build`’s `-o` switch to give the symlink another name.
+
+Nix has transactional semantics. Once a build finishes successfully, Nix
+makes a note of this in its database: it registers that the path denoted
+by out is now “valid”. If you try to build the derivation again, Nix
+will see that the path is already valid and finish immediately. If a
+build fails, either because it returns a non-zero exit code, because Nix
+or the builder are killed, or because the machine crashes, then the
+output paths will not be registered as valid. If you try to build the
+derivation again, Nix will remove the output paths if they exist (e.g.,
+because the builder died half-way through `make
+install`) and try again. Note that there is no “negative caching”: Nix
+doesn't remember that a build failed, and so a failed build can always
+be repeated. This is because Nix cannot distinguish between permanent
+failures (e.g., a compiler error due to a syntax error in the source)
+and transient failures (e.g., a disk full condition).
+
+Nix also performs locking. If you run multiple Nix builds
+simultaneously, and they try to build the same derivation, the first Nix
+instance that gets there will perform the build, while the others block
+(or perform other derivations if available) until the build finishes:
+
+    $ nix-build -A hello
+    waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'
+
+So it is always safe to run multiple instances of Nix in parallel (which
+isn’t the case with, say, `make`).
diff --git a/doc/manual/src/expressions/simple-expression.md b/doc/manual/src/expressions/simple-expression.md
new file mode 100644
index 000000000..9023f97a4
--- /dev/null
+++ b/doc/manual/src/expressions/simple-expression.md
@@ -0,0 +1,27 @@
+# A Simple Nix Expression
+
+This section shows how to add and test the [GNU Hello
+package](http://www.gnu.org/software/hello/hello.html) to the Nix
+Packages collection. Hello is a program that prints out the text “Hello,
+world\!”.
+
+To add a package to the Nix Packages collection, you generally need to
+do three things:
+
+1.  Write a Nix expression for the package. This is a file that
+    describes all the inputs involved in building the package, such as
+    dependencies, sources, and so on.
+
+2.  Write a *builder*. This is a shell script\[1\] that actually builds
+    the package from the inputs.
+
+3.  Add the package to the file `pkgs/top-level/all-packages.nix`. The
+    Nix expression written in the first step is a *function*; it
+    requires other packages in order to build it. In this step you put
+    it all together, i.e., you call the function with the right
+    arguments to build the actual package.
+
+<!-- end list -->
+
+1.  In fact, it can be written in any language, but typically it's a
+    `bash` shell script.
diff --git a/doc/manual/src/expressions/writing-nix-expressions.md b/doc/manual/src/expressions/writing-nix-expressions.md
new file mode 100644
index 000000000..5664108e7
--- /dev/null
+++ b/doc/manual/src/expressions/writing-nix-expressions.md
@@ -0,0 +1,12 @@
+This chapter shows you how to write Nix expressions, which instruct Nix
+how to build packages. It starts with a simple example (a Nix expression
+for GNU Hello), and then moves on to a more in-depth look at the Nix
+expression language.
+
+> **Note**
+> 
+> This chapter is mostly about the Nix expression language. For more
+> extensive information on adding packages to the Nix Packages
+> collection (such as functions in the standard environment and coding
+> conventions), please consult [its
+> manual](http://nixos.org/nixpkgs/manual/).
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
new file mode 100644
index 000000000..c8bdf4fd2
--- /dev/null
+++ b/doc/manual/src/glossary.md
@@ -0,0 +1,102 @@
+# Glossary
+
+  - derivation  
+    A description of a build action. The result of a derivation is a
+    store object. Derivations are typically specified in Nix expressions
+    using the [`derivation` primitive](#ssec-derivation). These are
+    translated into low-level *store derivations* (implicitly by
+    `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
+
+  - store  
+    The location in the file system where store objects live. Typically
+    `/nix/store`.
+
+  - store path  
+    The location in the file system of a store object, i.e., an
+    immediate child of the Nix store directory.
+
+  - store object  
+    A file that is an immediate child of the Nix store directory. These
+    can be regular files, but also entire directory trees. Store objects
+    can be sources (objects copied from outside of the store),
+    derivation outputs (objects produced by running a build action), or
+    derivations (files describing a build action).
+
+  - substitute  
+    A substitute is a command invocation stored in the Nix database that
+    describes how to build a store object, bypassing the normal build
+    mechanism (i.e., derivations). Typically, the substitute builds the
+    store object by downloading a pre-built version of the store object
+    from some server.
+
+  - purity  
+    The assumption that equal Nix derivations when run always produce
+    the same output. This cannot be guaranteed in general (e.g., a
+    builder can rely on external inputs such as the network or the
+    system time) but the Nix model assumes it.
+
+  - Nix expression  
+    A high-level description of software packages and compositions
+    thereof. Deploying software using Nix entails writing Nix
+    expressions for your packages. Nix expressions are translated to
+    derivations that are stored in the Nix store. These derivations can
+    then be built.
+
+  - reference  
+    A store path `P` is said to have a reference to a store path `Q` if
+    the store object at `P` contains the path `Q` somewhere. The
+    *references* of a store path are the set of store paths to which it
+    has a reference.
+    
+    A derivation can reference other derivations and sources (but not
+    output paths), whereas an output path only references other output
+    paths.
+
+  - reachable  
+    A store path `Q` is reachable from another store path `P` if `Q` is
+    in the [closure](#gloss-closure) of the
+    [references](#gloss-reference) relation.
+
+  - closure  
+    The closure of a store path is the set of store paths that are
+    directly or indirectly “reachable” from that store path; that is,
+    it’s the closure of the path under the
+    [references](#gloss-reference) relation. For a package, the closure
+    of its derivation is equivalent to the build-time dependencies,
+    while the closure of its output path is equivalent to its runtime
+    dependencies. For correct deployment it is necessary to deploy whole
+    closures, since otherwise at runtime files could be missing. The
+    command `nix-store -qR` prints out closures of store paths.
+    
+    As an example, if the store object at path `P` contains a reference
+    to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
+    references `R` then `R` is also in the closure of `P`.
+
+  - output path  
+    A store path produced by a derivation.
+
+  - deriver  
+    The deriver of an [output path](#gloss-output-path) is the store
+    derivation that built it.
+
+  - validity  
+    A store path is considered *valid* if it exists in the file system,
+    is listed in the Nix database as being valid, and if all paths in
+    its closure are also valid.
+
+  - user environment  
+    An automatically generated store object that consists of a set of
+    symlinks to “active” applications, i.e., other store paths. These
+    are generated automatically by [`nix-env`](#sec-nix-env). See
+    [???](#sec-profiles).
+
+  - profile  
+    A symlink to the current [user environment](#gloss-user-env) of a
+    user, e.g., `/nix/var/nix/profiles/default`.
+
+  - NAR  
+    A *N*ix *AR*chive. This is a serialisation of a path in the Nix
+    store. It can contain regular files, directories and symbolic links.
+    NARs are generated and unpacked using `nix-store --dump` and
+    `nix-store
+            --restore`.
diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
new file mode 100644
index 000000000..f8375822d
--- /dev/null
+++ b/doc/manual/src/hacking.md
@@ -0,0 +1,47 @@
+# Hacking
+
+This section provides some notes on how to hack on Nix. To get the
+latest version of Nix from GitHub:
+
+    $ git clone https://github.com/NixOS/nix.git
+    $ cd nix
+
+To build Nix for the current operating system/architecture use
+
+    $ nix-build
+
+or if you have a flakes-enabled nix:
+
+    $ nix build
+
+This will build `defaultPackage` attribute defined in the `flake.nix`
+file. To build for other platforms add one of the following suffixes to
+it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e.
+
+    $ nix-build -A defaultPackage.x86_64-linux
+
+To build all dependencies and start a shell in which all environment
+variables are set up so that those dependencies can be found:
+
+    $ nix-shell
+
+To build Nix itself in this shell:
+
+    [nix-shell]$ ./bootstrap.sh
+    [nix-shell]$ ./configure $configureFlags
+    [nix-shell]$ make -j $NIX_BUILD_CORES
+
+To install it in `$(pwd)/inst` and test it:
+
+    [nix-shell]$ make install
+    [nix-shell]$ make installcheck
+    [nix-shell]$ ./inst/bin/nix --version
+    nix (Nix) 2.4
+
+If you have a flakes-enabled nix you can replace:
+
+    $ nix-shell
+
+by:
+
+    $ nix develop
diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md
new file mode 100644
index 000000000..c498713de
--- /dev/null
+++ b/doc/manual/src/installation/building-source.md
@@ -0,0 +1,34 @@
+# Building Nix from Source
+
+After unpacking or checking out the Nix sources, issue the following
+commands:
+
+    $ ./configure options...
+    $ make
+    $ make install
+
+Nix requires GNU Make so you may need to invoke `gmake` instead.
+
+When building from the Git repository, these should be preceded by the
+command:
+
+    $ ./bootstrap.sh
+
+The installation path can be specified by passing the `--prefix=prefix`
+to `configure`. The default installation directory is `/usr/local`. You
+can change this to any location you like. You must have write permission
+to the prefix path.
+
+Nix keeps its *store* (the place where packages are stored) in
+`/nix/store` by default. This can be changed using
+`--with-store-dir=path`.
+
+> **Warning**
+> 
+> It is best *not* to change the Nix store from its default, since doing
+> so makes it impossible to use pre-built binaries from the standard
+> Nixpkgs channels — that is, all packages will need to be built from
+> source.
+
+Nix keeps state (such as its database and log files) in `/nix/var` by
+default. This can be changed using `--localstatedir=path`.
diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md
new file mode 100644
index 000000000..7946ac437
--- /dev/null
+++ b/doc/manual/src/installation/env-variables.md
@@ -0,0 +1,56 @@
+# Environment Variables
+
+To use Nix, some environment variables should be set. In particular,
+PATH should contain the directories `prefix/bin` and
+`~/.nix-profile/bin`. The first directory contains the Nix tools
+themselves, while `~/.nix-profile` is a symbolic link to the current
+*user environment* (an automatically generated package consisting of
+symlinks to installed packages). The simplest way to set the required
+environment variables is to include the file
+`prefix/etc/profile.d/nix.sh` in your `~/.profile` (or similar), like
+this:
+
+    source prefix/etc/profile.d/nix.sh
+
+# NIX\_SSL\_CERT\_FILE
+
+If you need to specify a custom certificate bundle to account for an
+HTTPS-intercepting man in the middle proxy, you must specify the path to
+the certificate bundle in the environment variable NIX\_SSL\_CERT\_FILE.
+
+If you don't specify a NIX\_SSL\_CERT\_FILE manually, Nix will install
+and use its own certificate bundle.
+
+Set the environment variable and install Nix
+
+    $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+    $ sh <(curl -L https://nixos.org/nix/install)
+
+In the shell profile and rc files (for example, `/etc/bashrc`,
+`/etc/zshrc`), add the following line:
+
+    export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+
+> **Note**
+> 
+> You must not add the export and then do the install, as the Nix
+> installer will detect the presense of Nix configuration, and abort.
+
+## NIX\_SSL\_CERT\_FILE with macOS and the Nix daemon
+
+On macOS you must specify the environment variable for the Nix daemon
+service, then restart it:
+
+    $ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt
+    $ sudo launchctl kickstart -k system/org.nixos.nix-daemon
+
+## Proxy Environment Variables
+
+The Nix installer has special handling for these proxy-related
+environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`,
+`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`.
+
+If any of these variables are set when running the Nix installer, then
+the installer will create an override file at
+`/etc/systemd/system/nix-daemon.service.d/override.conf` so `nix-daemon`
+will use them.
diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/installation.md
new file mode 100644
index 000000000..b40c5b95f
--- /dev/null
+++ b/doc/manual/src/installation/installation.md
@@ -0,0 +1,2 @@
+This section describes how to install and configure Nix for first-time
+use.
diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
new file mode 100644
index 000000000..d4e412e67
--- /dev/null
+++ b/doc/manual/src/installation/installing-binary.md
@@ -0,0 +1,273 @@
+# Installing a Binary Distribution
+
+If you are using Linux or macOS versions up to 10.14 (Mojave), the
+easiest way to install Nix is to run the following command:
+
+``` 
+  $ sh <(curl -L https://nixos.org/nix/install)
+```
+
+If you're using macOS 10.15 (Catalina) or newer, consult [the macOS
+installation instructions](#sect-macos-installation) before installing.
+
+As of Nix 2.1.0, the Nix installer will always default to creating a
+single-user installation, however opting in to the multi-user
+installation is highly recommended.
+
+# Single User Installation
+
+To explicitly select a single-user installation on your system:
+
+``` 
+  sh <(curl -L https://nixos.org/nix/install) --no-daemon
+```
+
+This will perform a single-user installation of Nix, meaning that `/nix`
+is owned by the invoking user. You should run this under your usual user
+account, *not* as root. The script will invoke `sudo` to create `/nix`
+if it doesn’t already exist. If you don’t have `sudo`, you should
+manually create `/nix` first as root, e.g.:
+
+    $ mkdir /nix
+    $ chown alice /nix
+
+The install script will modify the first writable file from amongst
+`.bash_profile`, `.bash_login` and `.profile` to source
+`~/.nix-profile/etc/profile.d/nix.sh`. You can set the
+NIX\_INSTALLER\_NO\_MODIFY\_PROFILE environment variable before
+executing the install script to disable this behaviour.
+
+You can uninstall Nix simply by running:
+
+    $ rm -rf /nix
+
+# Multi User Installation
+
+The multi-user Nix installation creates system users, and a system
+service for the Nix daemon.
+
+  - Linux running systemd, with SELinux disabled
+
+  - macOS
+
+You can instruct the installer to perform a multi-user installation on
+your system:
+
+    sh <(curl -L https://nixos.org/nix/install) --daemon
+
+The multi-user installation of Nix will create build users between the
+user IDs 30001 and 30032, and a group with the group ID 30000. You
+should run this under your usual user account, *not* as root. The script
+will invoke `sudo` as needed.
+
+> **Note**
+> 
+> If you need Nix to use a different group ID or user ID set, you will
+> have to download the tarball manually and [edit the install
+> script](#sect-nix-install-binary-tarball).
+
+The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
+The installer will first back up these files with a `.backup-before-nix`
+extension. The installer will also create `/etc/profile.d/nix.sh`.
+
+You can uninstall Nix with the following commands:
+
+    sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+    
+    # If you are on Linux with systemd, you will need to run:
+    sudo systemctl stop nix-daemon.socket
+    sudo systemctl stop nix-daemon.service
+    sudo systemctl disable nix-daemon.socket
+    sudo systemctl disable nix-daemon.service
+    sudo systemctl daemon-reload
+    
+    # If you are on macOS, you will need to run:
+    sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+    sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+
+There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
+and `/etc/zshrc` which you may remove.
+
+# macOS Installation
+
+Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
+This means `/nix` can no longer live on your system volume, and that
+you'll need a workaround to install Nix.
+
+The recommended approach, which creates an unencrypted APFS volume for
+your Nix store and a "synthetic" empty directory to mount it over at
+`/nix`, is least likely to impair Nix or your system.
+
+> **Note**
+> 
+> With all separate-volume approaches, it's possible something on your
+> system (particularly daemons/services and restored apps) may need
+> access to your Nix store before the volume is mounted. Adding
+> additional encryption makes this more likely.
+
+If you're using a recent Mac with a [T2
+chip](https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf),
+your drive will still be encrypted at rest (in which case "unencrypted"
+is a bit of a misnomer). To use this approach, just install Nix with:
+
+    $ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume
+
+If you don't like the sound of this, you'll want to weigh the other
+approaches and tradeoffs detailed in this section.
+
+> **Note**
+> 
+> All of the known workarounds have drawbacks, but we hope better
+> solutions will be available in the future. Some that we have our eye
+> on are:
+> 
+> 1.  A true firmlink would enable the Nix store to live on the primary
+>     data volume without the build problems caused by the symlink
+>     approach. End users cannot currently create true firmlinks.
+> 
+> 2.  If the Nix store volume shared FileVault encryption with the
+>     primary data volume (probably by using the same volume group and
+>     role), FileVault encryption could be easily supported by the
+>     installer without requiring manual setup by each user.
+
+## Change the Nix store path prefix
+
+Changing the default prefix for the Nix store is a simple approach which
+enables you to leave it on your root volume, where it can take full
+advantage of FileVault encryption if enabled. Unfortunately, this
+approach also opts your device out of some benefits that are enabled by
+using the same prefix across systems:
+
+  - Your system won't be able to take advantage of the binary cache
+    (unless someone is able to stand up and support duplicate caching
+    infrastructure), which means you'll spend more time waiting for
+    builds.
+
+  - It's harder to build and deploy packages to Linux systems.
+
+It would also possible (and often requested) to just apply this change
+ecosystem-wide, but it's an intrusive process that has side effects we
+want to avoid for now.
+
+## Use a separate encrypted volume
+
+If you like, you can also add encryption to the recommended approach
+taken by the installer. You can do this by pre-creating an encrypted
+volume before you run the installer--or you can run the installer and
+encrypt the volume it creates later.
+
+In either case, adding encryption to a second volume isn't quite as
+simple as enabling FileVault for your boot volume. Before you dive in,
+there are a few things to weigh:
+
+1.  The additional volume won't be encrypted with your existing
+    FileVault key, so you'll need another mechanism to decrypt the
+    volume.
+
+2.  You can store the password in Keychain to automatically decrypt the
+    volume on boot--but it'll have to wait on Keychain and may not mount
+    before your GUI apps restore. If any of your launchd agents or apps
+    depend on Nix-installed software (for example, if you use a
+    Nix-installed login shell), the restore may fail or break.
+    
+    On a case-by-case basis, you may be able to work around this problem
+    by using `wait4path` to block execution until your executable is
+    available.
+    
+    It's also possible to decrypt and mount the volume earlier with a
+    login hook--but this mechanism appears to be deprecated and its
+    future is unclear.
+
+3.  You can hard-code the password in the clear, so that your store
+    volume can be decrypted before Keychain is available.
+
+If you are comfortable navigating these tradeoffs, you can encrypt the
+volume with something along the lines of:
+
+    alice$ diskutil apfs enableFileVault /nix -user disk
+
+## Symlink the Nix store to a custom location
+
+Another simple approach is using `/etc/synthetic.conf` to symlink the
+Nix store to the data volume. This option also enables your store to
+share any configured FileVault encryption. Unfortunately, builds that
+resolve the symlink may leak the canonical path or even fail.
+
+Because of these downsides, we can't recommend this approach.
+
+## Notes on the recommended approach
+
+This section goes into a little more detail on the recommended approach.
+You don't need to understand it to run the installer, but it can serve
+as a helpful reference if you run into trouble.
+
+1.  In order to compose user-writable locations into the new read-only
+    system root, Apple introduced a new concept called `firmlinks`,
+    which it describes as a "bi-directional wormhole" between two
+    filesystems. You can see the current firmlinks in
+    `/usr/share/firmlinks`. Unfortunately, firmlinks aren't (currently?)
+    user-configurable.
+    
+    For special cases like NFS mount points or package manager roots,
+    [synthetic.conf(5)](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html)
+    supports limited user-controlled file-creation (of symlinks, and
+    synthetic empty directories) at `/`. To create a synthetic empty
+    directory for mounting at `/nix`, add the following line to
+    `/etc/synthetic.conf` (create it if necessary):
+    
+        nix
+
+2.  This configuration is applied at boot time, but you can use
+    `apfs.util` to trigger creation (not deletion) of new entries
+    without a reboot:
+    
+        alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+
+3.  Create the new APFS volume with diskutil:
+    
+        alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
+
+4.  Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't
+    already have other entries, it should look something like:
+    
+        #
+        # Warning - this file should only be modified with vifs(8)
+        #
+        # Failure to do so is unsupported and may be destructive.
+        #
+        LABEL=Nix\040Store /nix apfs rw,nobrowse
+    
+    The nobrowse setting will keep Spotlight from indexing this volume,
+    and keep it from showing up on your desktop.
+
+# Installing a pinned Nix version from a URL
+
+NixOS.org hosts version-specific installation URLs for all Nix versions
+since 1.11.16, at `https://releases.nixos.org/nix/nix-version/install`.
+
+These install scripts can be used the same as the main NixOS.org
+installation script:
+
+``` 
+  sh <(curl -L https://nixos.org/nix/install)
+```
+
+In the same directory of the install script are sha256 sums, and gpg
+signature files.
+
+# Installing from a binary tarball
+
+You can also download a binary tarball that contains Nix and all its
+dependencies. (This is what the install script at
+<https://nixos.org/nix/install> does automatically.) You should unpack
+it somewhere (e.g. in `/tmp`), and then run the script named `install`
+inside the binary tarball:
+
+    alice$ cd /tmp
+    alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
+    alice$ cd nix-1.8-x86_64-darwin
+    alice$ ./install
+
+If you need to edit the multi-user installation script to use different
+group ID or a different user ID range, modify the variables set in the
+file named `install-multi-user`.
diff --git a/doc/manual/src/installation/installing-source.md b/doc/manual/src/installation/installing-source.md
new file mode 100644
index 000000000..e52d38a03
--- /dev/null
+++ b/doc/manual/src/installation/installing-source.md
@@ -0,0 +1,4 @@
+# Installing Nix from Source
+
+If no binary package is available, you can download and compile a source
+distribution.
diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/src/installation/multi-user.md
new file mode 100644
index 000000000..17286fdc5
--- /dev/null
+++ b/doc/manual/src/installation/multi-user.md
@@ -0,0 +1,63 @@
+# Multi-User Mode
+
+To allow a Nix store to be shared safely among multiple users, it is
+important that users are not able to run builders that modify the Nix
+store or database in arbitrary ways, or that interfere with builds
+started by other users. If they could do so, they could install a Trojan
+horse in some package and compromise the accounts of other users.
+
+To prevent this, the Nix store and database are owned by some privileged
+user (usually `root`) and builders are executed under special user
+accounts (usually named `nixbld1`, `nixbld2`, etc.). When a unprivileged
+user runs a Nix command, actions that operate on the Nix store (such as
+builds) are forwarded to a *Nix daemon* running under the owner of the
+Nix store/database that performs the operation.
+
+> **Note**
+> 
+> Multi-user mode has one important limitation: only root and a set of
+> trusted users specified in `nix.conf` can specify arbitrary binary
+> caches. So while unprivileged users may install packages from
+> arbitrary Nix expressions, they may not get pre-built binaries.
+
+The *build users* are the special UIDs under which builds are performed.
+They should all be members of the *build users group* `nixbld`. This
+group should have no other members. The build users should not be
+members of any other group. On Linux, you can create the group and users
+as follows:
+
+    $ groupadd -r nixbld
+    $ for n in $(seq 1 10); do useradd -c "Nix build user $n" \
+        -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \
+        nixbld$n; done
+
+This creates 10 build users. There can never be more concurrent builds
+than the number of build users, so you may want to increase this if you
+expect to do many builds at the same time.
+
+The [Nix daemon](#sec-nix-daemon) should be started as follows (as
+`root`):
+
+    $ nix-daemon
+
+You’ll want to put that line somewhere in your system’s boot scripts.
+
+To let unprivileged users use the daemon, they should set the
+[NIX\_REMOTE environment variable](#envar-remote) to `daemon`. So you
+should put a line like
+
+    export NIX_REMOTE=daemon
+
+into the users’ login scripts.
+
+To limit which users can perform Nix operations, you can use the
+permissions on the directory `/nix/var/nix/daemon-socket`. For instance,
+if you want to restrict the use of Nix to the members of a group called
+`nix-users`, do
+
+    $ chgrp nix-users /nix/var/nix/daemon-socket
+    $ chmod ug=rwx,o= /nix/var/nix/daemon-socket
+
+This way, users who are not in the `nix-users` group cannot connect to
+the Unix domain socket `/nix/var/nix/daemon-socket/socket`, so they
+cannot perform Nix operations.
diff --git a/doc/manual/src/installation/nix-security.md b/doc/manual/src/installation/nix-security.md
new file mode 100644
index 000000000..1e9036b68
--- /dev/null
+++ b/doc/manual/src/installation/nix-security.md
@@ -0,0 +1,15 @@
+# Security
+
+Nix has two basic security models. First, it can be used in “single-user
+mode”, which is similar to what most other package management tools do:
+there is a single user (typically root) who performs all package
+management operations. All other users can then use the installed
+packages, but they cannot perform package management operations
+themselves.
+
+Alternatively, you can configure Nix in “multi-user mode”. In this
+model, all users can perform package management operations — for
+instance, every user can install software without requiring root
+privileges. Nix ensures that this is secure. For instance, it’s not
+possible for one user to overwrite a package used by another user with a
+Trojan horse.
diff --git a/doc/manual/src/installation/obtaining-source.md b/doc/manual/src/installation/obtaining-source.md
new file mode 100644
index 000000000..2937130cf
--- /dev/null
+++ b/doc/manual/src/installation/obtaining-source.md
@@ -0,0 +1,16 @@
+# Obtaining a Source Distribution
+
+The source tarball of the most recent stable release can be downloaded
+from the [Nix homepage](http://nixos.org/nix/download.html). You can
+also grab the [most recent development
+release](http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents).
+
+Alternatively, the most recent sources of Nix can be obtained from its
+[Git repository](https://github.com/NixOS/nix). For example, the
+following command will check out the latest revision into a directory
+called `nix`:
+
+    $ git clone https://github.com/NixOS/nix
+
+Likewise, specific releases can be obtained from the
+[tags](https://github.com/NixOS/nix/tags) of the repository.
diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
new file mode 100644
index 000000000..311961fe9
--- /dev/null
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -0,0 +1,80 @@
+# Prerequisites
+
+  - GNU Autoconf (<https://www.gnu.org/software/autoconf/>) and the
+    autoconf-archive macro collection
+    (<https://www.gnu.org/software/autoconf-archive/>). These are only
+    needed to run the bootstrap script, and are not necessary if your
+    source distribution came with a pre-built `./configure` script.
+
+  - GNU Make.
+
+  - Bash Shell. The `./configure` script relies on bashisms, so Bash is
+    required.
+
+  - A version of GCC or Clang that supports C++17.
+
+  - `pkg-config` to locate dependencies. If your distribution does not
+    provide it, you can get it from
+    <http://www.freedesktop.org/wiki/Software/pkg-config>.
+
+  - The OpenSSL library to calculate cryptographic hashes. If your
+    distribution does not provide it, you can get it from
+    <https://www.openssl.org>.
+
+  - The `libbrotlienc` and `libbrotlidec` libraries to provide
+    implementation of the Brotli compression algorithm. They are
+    available for download from the official repository
+    <https://github.com/google/brotli>.
+
+  - The bzip2 compressor program and the `libbz2` library. Thus you must
+    have bzip2 installed, including development headers and libraries.
+    If your distribution does not provide these, you can obtain bzip2
+    from
+    <https://web.archive.org/web/20180624184756/http://www.bzip.org/>.
+
+  - `liblzma`, which is provided by XZ Utils. If your distribution does
+    not provide this, you can get it from <https://tukaani.org/xz/>.
+
+  - cURL and its library. If your distribution does not provide it, you
+    can get it from <https://curl.haxx.se/>.
+
+  - The SQLite embedded database library, version 3.6.19 or higher. If
+    your distribution does not provide it, please install it from
+    <http://www.sqlite.org/>.
+
+  - The [Boehm garbage collector](http://www.hboehm.info/gc/) to reduce
+    the evaluator’s memory consumption (optional). To enable it, install
+    `pkgconfig` and the Boehm garbage collector, and pass the flag
+    `--enable-gc` to `configure`.
+
+  - The `boost` library of version 1.66.0 or higher. It can be obtained
+    from the official web site <https://www.boost.org/>.
+
+  - The `editline` library of version 1.14.0 or higher. It can be
+    obtained from the its repository
+    <https://github.com/troglobit/editline>.
+
+  - The `xmllint` and `xsltproc` programs to build this manual and the
+    man-pages. These are part of the `libxml2` and `libxslt` packages,
+    respectively. You also need the [DocBook XSL
+    stylesheets](http://docbook.sourceforge.net/projects/xsl/) and
+    optionally the [DocBook 5.0 RELAX NG
+    schemas](http://www.docbook.org/schemas/5x). Note that these are
+    only required if you modify the manual sources or when you are
+    building from the Git repository.
+
+  - Recent versions of Bison and Flex to build the parser. (This is
+    because Nix needs GLR support in Bison and reentrancy support in
+    Flex.) For Bison, you need version 2.6, which can be obtained from
+    the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex,
+    you need version 2.5.35, which is available on
+    [SourceForge](http://lex.sourceforge.net/). Slightly older versions
+    may also work, but ancient versions like the ubiquitous 2.5.4a
+    won't. Note that these are only required if you modify the parser or
+    when you are building from the Git repository.
+
+  - The `libseccomp` is used to provide syscall filtering on Linux. This
+    is an optional dependency and can be disabled passing a
+    `--disable-seccomp-sandboxing` option to the `configure` script (Not
+    recommended unless your system doesn't support `libseccomp`). To get
+    the library, visit <https://github.com/seccomp/libseccomp>.
diff --git a/doc/manual/src/installation/single-user.md b/doc/manual/src/installation/single-user.md
new file mode 100644
index 000000000..f9a3b26ed
--- /dev/null
+++ b/doc/manual/src/installation/single-user.md
@@ -0,0 +1,9 @@
+# Single-User Mode
+
+In single-user mode, all Nix operations that access the database in
+`prefix/var/nix/db` or modify the Nix store in `prefix/store` must be
+performed under the user ID that owns those directories. This is
+typically root. (If you install from RPM packages, that’s in fact the
+default ownership.) However, on single-user machines, it is often
+convenient to `chown` those directories to your normal user account so
+that you don’t have to `su` to root all the time.
diff --git a/doc/manual/src/installation/supported-platforms.md b/doc/manual/src/installation/supported-platforms.md
new file mode 100644
index 000000000..8ef1f0e78
--- /dev/null
+++ b/doc/manual/src/installation/supported-platforms.md
@@ -0,0 +1,7 @@
+# Supported Platforms
+
+Nix is currently supported on the following platforms:
+
+  - Linux (i686, x86\_64, aarch64).
+
+  - macOS (x86\_64).
diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md
new file mode 100644
index 000000000..24efc4681
--- /dev/null
+++ b/doc/manual/src/installation/upgrading.md
@@ -0,0 +1,14 @@
+# Upgrading Nix
+
+Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c
+'nix-channel --update &&
+nix-env -iA nixpkgs.nix &&
+launchctl remove org.nixos.nix-daemon &&
+launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'`
+
+Single-user installations of Nix should run this: `nix-channel --update;
+nix-env -iA nixpkgs.nix nixpkgs.cacert`
+
+Multi-user Nix users on Linux should run this with sudo: `nix-channel
+--update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl
+daemon-reload; systemctl restart nix-daemon`
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
new file mode 100644
index 000000000..f5c550fd7
--- /dev/null
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -0,0 +1,148 @@
+# Basic Package Management
+
+The main command for package management is [`nix-env`](#sec-nix-env).
+You can use it to install, upgrade, and erase packages, and to query
+what packages are installed or are available for installation.
+
+In Nix, different users can have different “views” on the set of
+installed applications. That is, there might be lots of applications
+present on the system (possibly in many different versions), but users
+can have a specific selection of those active — where “active” just
+means that it appears in a directory in the user’s PATH. Such a view on
+the set of installed applications is called a *user environment*, which
+is just a directory tree consisting of symlinks to the files of the
+active applications.
+
+Components are installed from a set of *Nix expressions* that tell Nix
+how to build those packages, including, if necessary, their
+dependencies. There is a collection of Nix expressions called the
+Nixpkgs package collection that contains packages ranging from basic
+development stuff such as GCC and Glibc, to end-user applications like
+Mozilla Firefox. (Nix is however not tied to the Nixpkgs package
+collection; you could write your own Nix expressions based on Nixpkgs,
+or completely new ones.)
+
+You can manually download the latest version of Nixpkgs from
+<http://nixos.org/nixpkgs/download.html>. However, it’s much more
+convenient to use the Nixpkgs *channel*, since it makes it easy to stay
+up to date with new versions of Nixpkgs. (Channels are described in more
+detail in [???](#sec-channels).) Nixpkgs is automatically added to your
+list of “subscribed” channels when you install Nix. If this is not the
+case for some reason, you can add it as follows:
+
+    $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    $ nix-channel --update
+
+> **Note**
+> 
+> On NixOS, you’re automatically subscribed to a NixOS channel
+> corresponding to your NixOS major release (e.g.
+> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical
+> to the Nixpkgs channel, except that it contains only Linux binaries
+> and is updated only if a set of regression tests succeed.
+
+You can view the set of available packages in Nixpkgs:
+
+    $ nix-env -qa
+    aterm-2.2
+    bash-3.0
+    binutils-2.15
+    bison-1.875d
+    blackdown-1.4.2
+    bzip2-1.0.2
+    …
+
+The flag `-q` specifies a query operation, and `-a` means that you want
+to show the “available” (i.e., installable) packages, as opposed to the
+installed packages. If you downloaded Nixpkgs yourself, or if you
+checked it out from GitHub, then you need to pass the path to your
+Nixpkgs tree using the `-f` flag:
+
+    $ nix-env -qaf /path/to/nixpkgs
+
+where /path/to/nixpkgs is where you’ve unpacked or checked out Nixpkgs.
+
+You can select specific packages by name:
+
+    $ nix-env -qa firefox
+    firefox-34.0.5
+    firefox-with-plugins-34.0.5
+
+and using regular expressions:
+
+    $ nix-env -qa 'firefox.*'
+
+It is also possible to see the *status* of available packages, i.e.,
+whether they are installed into the user environment and/or present in
+the system:
+
+    $ nix-env -qas
+    …
+    -PS bash-3.0
+    --S binutils-2.15
+    IPS bison-1.875d
+    …
+
+The first character (`I`) indicates whether the package is installed in
+your current user environment. The second (`P`) indicates whether it is
+present on your system (in which case installing it into your user
+environment would be a very quick operation). The last one (`S`)
+indicates whether there is a so-called *substitute* for the package,
+which is Nix’s mechanism for doing binary deployment. It just means that
+Nix knows that it can fetch a pre-built package from somewhere
+(typically a network server) instead of building it locally.
+
+You can install a package using `nix-env -i`. For instance,
+
+    $ nix-env -i subversion
+
+will install the package called `subversion` (which is, of course, the
+[Subversion version management system](http://subversion.tigris.org/)).
+
+> **Note**
+> 
+> When you ask Nix to install a package, it will first try to get it in
+> pre-compiled form from a *binary cache*. By default, Nix will use the
+> binary cache <https://cache.nixos.org>; it contains binaries for most
+> packages in Nixpkgs. Only if no binary is available in the binary
+> cache, Nix will build the package from source. So if `nix-env
+> -i subversion` results in Nix building stuff from source, then either
+> the package is not built for your platform by the Nixpkgs build
+> servers, or your version of Nixpkgs is too old or too new. For
+> instance, if you have a very recent checkout of Nixpkgs, then the
+> Nixpkgs build servers may not have had a chance to build everything
+> and upload the resulting binaries to <https://cache.nixos.org>. The
+> Nixpkgs channel is only updated after all binaries have been uploaded
+> to the cache, so if you stick to the Nixpkgs channel (rather than
+> using a Git checkout of the Nixpkgs tree), you will get binaries for
+> most packages.
+
+Naturally, packages can also be uninstalled:
+
+    $ nix-env -e subversion
+
+Upgrading to a new version is just as easy. If you have a new release of
+Nix Packages, you can do:
+
+    $ nix-env -u subversion
+
+This will *only* upgrade Subversion if there is a “newer” version in the
+new set of Nix expressions, as defined by some pretty arbitrary rules
+regarding ordering of version numbers (which generally do what you’d
+expect of them). To just unconditionally replace Subversion with
+whatever version is in the Nix expressions, use `-i` instead of `-u`;
+`-i` will remove whatever version is already installed.
+
+You can also upgrade all packages for which there are newer versions:
+
+    $ nix-env -u
+
+Sometimes it’s useful to be able to ask what `nix-env` would do, without
+actually doing it. For instance, to find out what packages would be
+upgraded by `nix-env -u`, you can do
+
+    $ nix-env -u --dry-run
+    (dry run; not doing anything)
+    upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
+    upgrading `graphviz-1.10' to `graphviz-1.12'
+    upgrading `coreutils-5.0' to `coreutils-5.2.1'
diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md
new file mode 100644
index 000000000..44f0da238
--- /dev/null
+++ b/doc/manual/src/package-management/binary-cache-substituter.md
@@ -0,0 +1,42 @@
+# Serving a Nix store via HTTP
+
+You can easily share the Nix store of a machine via HTTP. This allows
+other machines to fetch store paths from that machine to speed up
+installations. It uses the same *binary cache* mechanism that Nix
+usually uses to fetch pre-built binaries from <https://cache.nixos.org>.
+
+The daemon that handles binary cache requests via HTTP, `nix-serve`, is
+not part of the Nix distribution, but you can install it from Nixpkgs:
+
+    $ nix-env -i nix-serve
+
+You can then start the server, listening for HTTP connections on
+whatever port you like:
+
+    $ nix-serve -p 8080
+
+To check whether it works, try the following on the client:
+
+    $ curl http://avalon:8080/nix-cache-info
+
+which should print something like:
+
+    StoreDir: /nix/store
+    WantMassQuery: 1
+    Priority: 30
+
+On the client side, you can tell Nix to use your binary cache using
+`--option extra-binary-caches`, e.g.:
+
+    $ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
+
+The option `extra-binary-caches` tells Nix to use this binary cache in
+addition to your default caches, such as <https://cache.nixos.org>.
+Thus, for any path in the closure of Firefox, Nix will first check if
+the path is available on the server `avalon` or another binary caches.
+If not, it will fall back to building from source.
+
+You can also tell Nix to always use your binary cache by adding a line
+to the `nix.conf` configuration file like this:
+
+    binary-caches = http://avalon:8080/ https://cache.nixos.org/
diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md
new file mode 100644
index 000000000..8a3e92d70
--- /dev/null
+++ b/doc/manual/src/package-management/channels.md
@@ -0,0 +1,42 @@
+# Channels
+
+If you want to stay up to date with a set of packages, it’s not very
+convenient to manually download the latest set of Nix expressions for
+those packages and upgrade using `nix-env`. Fortunately, there’s a
+better way: *Nix channels*.
+
+A Nix channel is just a URL that points to a place that contains a set
+of Nix expressions and a manifest. Using the command
+[`nix-channel`](#sec-nix-channel) you can automatically stay up to date
+with whatever is available at that URL.
+
+To see the list of official NixOS channels, visit
+<https://nixos.org/channels>.
+
+You can “subscribe” to a channel using `nix-channel --add`, e.g.,
+
+    $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+
+subscribes you to a channel that always contains that latest version of
+the Nix Packages collection. (Subscribing really just means that the URL
+is added to the file `~/.nix-channels`, where it is read by subsequent
+calls to `nix-channel
+--update`.) You can “unsubscribe” using `nix-channel
+--remove`:
+
+    $ nix-channel --remove nixpkgs
+
+To obtain the latest Nix expressions available in a channel, do
+
+    $ nix-channel --update
+
+This downloads and unpacks the Nix expressions in every channel
+(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of
+each channel’s Nix expressions available by default to `nix-env`
+operations (via the symlink `~/.nix-defexpr/channels`). Consequently,
+you can then say
+
+    $ nix-env -u
+
+to upgrade all packages in your profile to the latest versions available
+in the subscribed channels.
diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md
new file mode 100644
index 000000000..6573862c0
--- /dev/null
+++ b/doc/manual/src/package-management/copy-closure.md
@@ -0,0 +1,34 @@
+# Copying Closures Via SSH
+
+The command `nix-copy-closure` copies a Nix store path along with all
+its dependencies to or from another machine via the SSH protocol. It
+doesn’t copy store paths that are already present on the target machine.
+For example, the following command copies Firefox with all its
+dependencies:
+
+    $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)
+
+See [???](#sec-nix-copy-closure) for details.
+
+With `nix-store
+--export` and `nix-store --import` you can write the closure of a store
+path (that is, the path and all its dependencies) to a file, and then
+unpack that file into another Nix store. For example,
+
+    $ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure
+
+writes the closure of Firefox to a file. You can then copy this file to
+another machine and install the closure:
+
+    $ nix-store --import < firefox.closure
+
+Any store paths in the closure that are already present in the target
+store are ignored. It is also possible to pipe the export into another
+command, e.g. to copy and install a closure directly to/on another
+machine:
+
+    $ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \
+        ssh alice@itchy.example.org "bunzip2 | nix-store --import"
+
+However, `nix-copy-closure` is generally more efficient because it only
+copies paths that are not already present in the target Nix store.
diff --git a/doc/manual/src/package-management/garbage-collection.md b/doc/manual/src/package-management/garbage-collection.md
new file mode 100644
index 000000000..4c8799dfe
--- /dev/null
+++ b/doc/manual/src/package-management/garbage-collection.md
@@ -0,0 +1,61 @@
+# Garbage Collection
+
+`nix-env` operations such as upgrades (`-u`) and uninstall (`-e`) never
+actually delete packages from the system. All they do (as shown above)
+is to create a new user environment that no longer contains symlinks to
+the “deleted” packages.
+
+Of course, since disk space is not infinite, unused packages should be
+removed at some point. You can do this by running the Nix garbage
+collector. It will remove from the Nix store any package not used
+(directly or indirectly) by any generation of any profile.
+
+Note however that as long as old generations reference a package, it
+will not be deleted. After all, we wouldn’t be able to do a rollback
+otherwise. So in order for garbage collection to be effective, you
+should also delete (some) old generations. Of course, this should only
+be done if you are certain that you will not need to roll back.
+
+To delete all old (non-current) generations of your current profile:
+
+    $ nix-env --delete-generations old
+
+Instead of `old` you can also specify a list of generations, e.g.,
+
+    $ nix-env --delete-generations 10 11 14
+
+To delete all generations older than a specified number of days (except
+the current generation), use the `d` suffix. For example,
+
+    $ nix-env --delete-generations 14d
+
+deletes all generations older than two weeks.
+
+After removing appropriate old generations you can run the garbage
+collector as follows:
+
+    $ nix-store --gc
+
+The behaviour of the gargage collector is affected by the
+`keep-derivations` (default: true) and `keep-outputs` (default: false)
+options in the Nix configuration file. The defaults will ensure that all
+derivations that are build-time dependencies of garbage collector roots
+will be kept and that all output paths that are runtime dependencies
+will be kept as well. All other derivations or paths will be collected.
+(This is usually what you want, but while you are developing it may make
+sense to keep outputs to ensure that rebuild times are quick.) If you
+are feeling uncertain, you can also first view what files would be
+deleted:
+
+    $ nix-store --gc --print-dead
+
+Likewise, the option `--print-live` will show the paths that *won’t* be
+deleted.
+
+There is also a convenient little utility `nix-collect-garbage`, which
+when invoked with the `-d` (`--delete-old`) switch deletes all old
+generations of all profiles in `/nix/var/nix/profiles`. So
+
+    $ nix-collect-garbage -d
+
+is a quick and easy way to clean up your system.
diff --git a/doc/manual/src/package-management/garbage-collector-roots.md b/doc/manual/src/package-management/garbage-collector-roots.md
new file mode 100644
index 000000000..6f4d48e80
--- /dev/null
+++ b/doc/manual/src/package-management/garbage-collector-roots.md
@@ -0,0 +1,16 @@
+# Garbage Collector Roots
+
+The roots of the garbage collector are all store paths to which there
+are symlinks in the directory `prefix/nix/var/nix/gcroots`. For
+instance, the following command makes the path
+`/nix/store/d718ef...-foo` a root of the collector:
+
+    $ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
+
+That is, after this command, the garbage collector will not remove
+`/nix/store/d718ef...-foo` or any of its dependencies.
+
+Subdirectories of `prefix/nix/var/nix/gcroots` are also searched for
+symlinks. Symlinks to non-store paths are followed and searched for
+roots, but symlinks to non-store paths *inside* the paths reached in
+that way are not followed to prevent infinite recursion.
diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/package-management.md
new file mode 100644
index 000000000..a2d078c16
--- /dev/null
+++ b/doc/manual/src/package-management/package-management.md
@@ -0,0 +1,4 @@
+This chapter discusses how to do package management with Nix, i.e., how
+to obtain, install, upgrade, and erase packages. This is the “user’s”
+perspective of the Nix system — people who want to *create* packages
+should consult [???](#chap-writing-nix-expressions).
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
new file mode 100644
index 000000000..c46adf538
--- /dev/null
+++ b/doc/manual/src/package-management/profiles.md
@@ -0,0 +1,119 @@
+# Profiles
+
+Profiles and user environments are Nix’s mechanism for implementing the
+ability to allow different users to have different configurations, and
+to do atomic upgrades and rollbacks. To understand how they work, it’s
+useful to know a bit about how Nix works. In Nix, packages are stored in
+unique locations in the *Nix store* (typically, `/nix/store`). For
+instance, a particular version of the Subversion package might be stored
+in a directory
+`/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/`, while
+another version might be stored in
+`/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long
+strings prefixed to the directory names are cryptographic hashes\[1\] of
+*all* inputs involved in building the package — sources, dependencies,
+compiler flags, and so on. So if two packages differ in any way, they
+end up in different locations in the file system, so they don’t
+interfere with each other. [figure\_title](#fig-user-environments) shows
+a part of a typical Nix store.
+
+![User environments](../figures/user-environments.png)
+
+Of course, you wouldn’t want to type
+
+    $ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn
+
+every time you want to run Subversion. Of course we could set up the
+PATH environment variable to include the `bin` directory of every
+package we want to use, but this is not very convenient since changing
+PATH doesn’t take effect for already existing processes. The solution
+Nix uses is to create directory trees of symlinks to *activated*
+packages. These are called *user environments* and they are packages
+themselves (though automatically generated by `nix-env`), so they too
+reside in the Nix store. For instance, in
+[figure\_title](#fig-user-environments) the user environment
+`/nix/store/0c1p5z4kda11...-user-env` contains a symlink to just
+Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be
+what we would obtain if we had done
+
+    $ nix-env -i subversion
+
+on a set of Nix expressions that contained Subversion 1.1.2.
+
+This doesn’t in itself solve the problem, of course; you wouldn’t want
+to type `/nix/store/0c1p5z4kda11...-user-env/bin/svn` either. That’s why
+there are symlinks outside of the store that point to the user
+environments in the store; for instance, the symlinks `default-42-link`
+and `default-43-link` in the example. These are called *generations*
+since every time you perform a `nix-env` operation, a new user
+environment is generated based on the current one. For instance,
+generation 43 was created from generation 42 when we did
+
+    $ nix-env -i subversion firefox
+
+on a set of Nix expressions that contained Firefox and a new version of
+Subversion.
+
+Generations are grouped together into *profiles* so that different users
+don’t interfere with each other if they don’t want to. For example:
+
+    $ ls -l /nix/var/nix/profiles/
+    ...
+    lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
+    lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
+    lrwxrwxrwx  1 eelco ... default -> default-43-link
+
+This shows a profile called `default`. The file `default` itself is
+actually a symlink that points to the current generation. When we do a
+`nix-env` operation, a new user environment and generation link are
+created based on the current one, and finally the `default` symlink is
+made to point at the new generation. This last step is atomic on Unix,
+which explains how we can do atomic upgrades. (Note that the
+building/installing of new packages doesn’t interfere in any way with
+old packages, since they are stored in different locations in the Nix
+store.)
+
+If you find that you want to undo a `nix-env` operation, you can just do
+
+    $ nix-env --rollback
+
+which will just make the current generation link point at the previous
+link. E.g., `default` would be made to point at `default-42-link`. You
+can also switch to a specific generation:
+
+    $ nix-env --switch-generation 43
+
+which in this example would roll forward to generation 43 again. You can
+also see all available generations:
+
+    $ nix-env --list-generations
+
+You generally wouldn’t have `/nix/var/nix/profiles/some-profile/bin` in
+your PATH. Rather, there is a symlink `~/.nix-profile` that points to
+your current profile. This means that you should put
+`~/.nix-profile/bin` in your PATH (and indeed, that’s what the
+initialisation script `/nix/etc/profile.d/nix.sh` does). This makes it
+easier to switch to a different profile. You can do that using the
+command `nix-env --switch-profile`:
+
+    $ nix-env --switch-profile /nix/var/nix/profiles/my-profile
+    
+    $ nix-env --switch-profile /nix/var/nix/profiles/default
+
+These commands switch to the `my-profile` and default profile,
+respectively. If the profile doesn’t exist, it will be created
+automatically. You should be careful about storing a profile in another
+location than the `profiles` directory, since otherwise it might not be
+used as a root of the garbage collector (see
+[???](#sec-garbage-collection)).
+
+All `nix-env` operations work on the profile pointed to by
+`~/.nix-profile`, but you can override this using the `--profile` option
+(abbreviation `-p`):
+
+    $ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
+
+This will *not* change the `~/.nix-profile` symlink.
+
+1.  160-bit truncations of SHA-256 hashes encoded in a base-32 notation,
+    to be precise.
diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md
new file mode 100644
index 000000000..4740b8b1c
--- /dev/null
+++ b/doc/manual/src/package-management/s3-substituter.md
@@ -0,0 +1,133 @@
+# Serving a Nix store via AWS S3 or S3-compatible Service
+
+Nix has built-in support for storing and fetching store paths from
+Amazon S3 and S3 compatible services. This uses the same *binary* cache
+mechanism that Nix usually uses to fetch prebuilt binaries from
+[cache.nixos.org](cache.nixos.org).
+
+The following options can be specified as URL parameters to the S3 URL:
+
+  - `profile`  
+    The name of the AWS configuration profile to use. By default Nix
+    will use the `default` profile.
+
+  - `region`  
+    The region of the S3 bucket. `us–east-1` by default.
+    
+    If your bucket is not in `us–east-1`, you should always explicitly
+    specify the region parameter.
+
+  - `endpoint`  
+    The URL to your S3-compatible service, for when not using Amazon S3.
+    Do not specify this value if you're using Amazon S3.
+    
+    > **Note**
+    > 
+    > This endpoint must support HTTPS and will use path-based
+    > addressing instead of virtual host based addressing.
+
+  - `scheme`  
+    The scheme used for S3 requests, `https` (default) or `http`. This
+    option allows you to disable HTTPS for binary caches which don't
+    support it.
+    
+    > **Note**
+    > 
+    > HTTPS should be used if the cache might contain sensitive
+    > information.
+
+In this example we will use the bucket named `example-nix-cache`.
+
+## Anonymous Reads to your S3-compatible binary cache
+
+If your binary cache is publicly accessible and does not require
+authentication, the simplest and easiest way to use Nix with your S3
+compatible binary cache is to use the HTTP URL for that cache.
+
+For AWS S3 the binary cache URL for example bucket will be exactly
+<https://example-nix-cache.s3.amazonaws.com> or
+<s3://example-nix-cache>. For S3 compatible binary caches, consult that
+cache's documentation.
+
+Your bucket will need the following bucket policy:
+
+    {
+        "Id": "DirectReads",
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "AllowDirectReads",
+                "Action": [
+                    "s3:GetObject",
+                    "s3:GetBucketLocation"
+                ],
+                "Effect": "Allow",
+                "Resource": [
+                    "arn:aws:s3:::example-nix-cache",
+                    "arn:aws:s3:::example-nix-cache/*"
+                ],
+                "Principal": "*"
+            }
+        ]
+    }
+
+## Authenticated Reads to your S3 binary cache
+
+For AWS S3 the binary cache URL for example bucket will be exactly
+<s3://example-nix-cache>.
+
+Nix will use the [default credential provider
+chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
+for authenticating requests to Amazon S3.
+
+Nix supports authenticated reads from Amazon S3 and S3 compatible binary
+caches.
+
+Your bucket will need a bucket policy allowing the desired users to
+perform the `s3:GetObject` and `s3:GetBucketLocation` action on all
+objects in the bucket. The anonymous policy in [Anonymous Reads to your
+S3-compatible binary cache](#ssec-s3-substituter-anonymous-reads) can be
+updated to have a restricted `Principal` to support this.
+
+## Authenticated Writes to your S3-compatible binary cache
+
+Nix support fully supports writing to Amazon S3 and S3 compatible
+buckets. The binary cache URL for our example bucket will be
+<s3://example-nix-cache>.
+
+Nix will use the [default credential provider
+chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
+for authenticating requests to Amazon S3.
+
+Your account will need the following IAM policy to upload to the cache:
+
+    {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Sid": "UploadToCache",
+          "Effect": "Allow",
+          "Action": [
+            "s3:AbortMultipartUpload",
+            "s3:GetBucketLocation",
+            "s3:GetObject",
+            "s3:ListBucket",
+            "s3:ListBucketMultipartUploads",
+            "s3:ListMultipartUploadParts",
+            "s3:PutObject"
+          ],
+          "Resource": [
+            "arn:aws:s3:::example-nix-cache",
+            "arn:aws:s3:::example-nix-cache/*"
+          ]
+        }
+      ]
+    }
+
+`nix copy --to
+'s3://example-nix-cache?profile=cache-upload&region=eu-west-2'
+nixpkgs.hello`
+
+`nix copy --to
+'s3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
+nixpkgs.hello`
diff --git a/doc/manual/src/package-management/sharing-packages.md b/doc/manual/src/package-management/sharing-packages.md
new file mode 100644
index 000000000..846ca6934
--- /dev/null
+++ b/doc/manual/src/package-management/sharing-packages.md
@@ -0,0 +1,6 @@
+# Sharing Packages Between Machines
+
+Sometimes you want to copy a package from one machine to another. Or,
+you want to install some packages and you know that another machine
+already has some or all of those packages or their dependencies. In that
+case there are mechanisms to quickly copy packages between machines.
diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md
new file mode 100644
index 000000000..482844c7c
--- /dev/null
+++ b/doc/manual/src/package-management/ssh-substituter.md
@@ -0,0 +1,52 @@
+# Serving a Nix store via SSH
+
+You can tell Nix to automatically fetch needed binaries from a remote
+Nix store via SSH. For example, the following installs Firefox,
+automatically fetching any store paths in Firefox’s closure if they are
+available on the server `avalon`:
+
+    $ nix-env -i firefox --substituters ssh://alice@avalon
+
+This works similar to the binary cache substituter that Nix usually
+uses, only using SSH instead of HTTP: if a store path `P` is needed, Nix
+will first check if it’s available in the Nix store on `avalon`. If not,
+it will fall back to using the binary cache substituter, and then to
+building from source.
+
+> **Note**
+> 
+> The SSH substituter currently does not allow you to enter an SSH
+> passphrase interactively. Therefore, you should use `ssh-add` to load
+> the decrypted private key into `ssh-agent`.
+
+You can also copy the closure of some store path, without installing it
+into your profile, e.g.
+
+    $ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon
+
+This is essentially equivalent to doing
+
+    $ nix-copy-closure --from alice@avalon /nix/store/m85bxg…-firefox-34.0.5
+
+You can use SSH’s *forced command* feature to set up a restricted user
+account for SSH substituter access, allowing read-only access to the
+local Nix store, but nothing more. For example, add the following lines
+to `sshd_config` to restrict the user `nix-ssh`:
+
+    Match User nix-ssh
+      AllowAgentForwarding no
+      AllowTcpForwarding no
+      PermitTTY no
+      PermitTunnel no
+      X11Forwarding no
+      ForceCommand nix-store --serve
+    Match All
+
+On NixOS, you can accomplish the same by adding the following to your
+`configuration.nix`:
+
+    nix.sshServe.enable = true;
+    nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+where the latter line lists the public keys of users that are allowed to
+connect.
diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md
new file mode 100644
index 000000000..b05d5ee0a
--- /dev/null
+++ b/doc/manual/src/release-notes/release-notes.md
@@ -0,0 +1 @@
+# Nix Release Notes
diff --git a/doc/manual/src/release-notes/rl-0.10.1.md b/doc/manual/src/release-notes/rl-0.10.1.md
new file mode 100644
index 000000000..e1ed6558a
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.10.1.md
@@ -0,0 +1,5 @@
+# Release 0.10.1 (2006-10-11)
+
+This release fixes two somewhat obscure bugs that occur when evaluating
+Nix expressions that are stored inside the Nix store (`NIX-67`). These
+do not affect most users.
diff --git a/doc/manual/src/release-notes/rl-0.10.md b/doc/manual/src/release-notes/rl-0.10.md
new file mode 100644
index 000000000..bf0e59fad
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.10.md
@@ -0,0 +1,212 @@
+# Release 0.10 (2006-10-06)
+
+> **Note**
+> 
+> This version of Nix uses Berkeley DB 4.4 instead of 4.3. The database
+> is upgraded automatically, but you should be careful not to use old
+> versions of Nix that still use Berkeley DB 4.3. In particular, if you
+> use a Nix installed through Nix, you should run
+> 
+>     $ nix-store --clear-substitutes
+> 
+> first.
+
+> **Warning**
+> 
+> Also, the database schema has changed slighted to fix a performance
+> issue (see below). When you run any Nix 0.10 command for the first
+> time, the database will be upgraded automatically. This is
+> irreversible.
+
+  - `nix-env` usability improvements:
+    
+      - An option `--compare-versions` (or `-c`) has been added to
+        `nix-env
+                                                        --query` to allow you to compare installed versions of packages
+        to available versions, or vice versa. An easy way to see if you
+        are up to date with what’s in your subscribed channels is
+        `nix-env -qc \*`.
+    
+      - `nix-env --query` now takes as arguments a list of package names
+        about which to show information, just like `--install`, etc.:
+        for example, `nix-env -q gcc`. Note that to show all
+        derivations, you need to specify `\*`.
+    
+      - `nix-env -i
+                                                        pkgname` will now install the highest available version of
+        pkgname, rather than installing all available versions (which
+        would probably give collisions) (`NIX-31`).
+    
+      - `nix-env (-i|-u) --dry-run` now shows exactly which missing
+        paths will be built or substituted.
+    
+      - `nix-env -qa --description` shows human-readable descriptions of
+        packages, provided that they have a `meta.description` attribute
+        (which most packages in Nixpkgs don’t have yet).
+
+  - New language features:
+    
+      - Reference scanning (which happens after each build) is much
+        faster and takes a constant amount of memory.
+    
+      - String interpolation. Expressions like
+        
+            "--with-freetype2-library=" + freetype + "/lib"
+        
+        can now be written as
+        
+            "--with-freetype2-library=${freetype}/lib"
+        
+        You can write arbitrary expressions within `${...}`, not just
+        identifiers.
+    
+      - Multi-line string literals.
+    
+      - String concatenations can now involve derivations, as in the
+        example `"--with-freetype2-library="
+                                                        + freetype + "/lib"`. This was not previously possible because
+        we need to register that a derivation that uses such a string is
+        dependent on `freetype`. The evaluator now properly propagates
+        this information. Consequently, the subpath operator (`~`) has
+        been deprecated.
+    
+      - Default values of function arguments can now refer to other
+        function arguments; that is, all arguments are in scope in the
+        default values (`NIX-45`).
+    
+      - Lots of new built-in primitives, such as functions for list
+        manipulation and integer arithmetic. See the manual for a
+        complete list. All primops are now available in the set
+        `builtins`, allowing one to test for the availability of primop
+        in a backwards-compatible way.
+    
+      - Real let-expressions: `let x = ...;
+                                                        ... z = ...; in ...`.
+
+  - New commands `nix-pack-closure` and `nix-unpack-closure` than can be
+    used to easily transfer a store path with all its dependencies to
+    another machine. Very convenient whenever you have some package on
+    your machine and you want to copy it somewhere else.
+
+  - XML support:
+    
+      - `nix-env -q --xml` prints the installed or available packages in
+        an XML representation for easy processing by other tools.
+    
+      - `nix-instantiate --eval-only
+                                                        --xml` prints an XML representation of the resulting term. (The
+        new flag `--strict` forces ‘deep’ evaluation of the result,
+        i.e., list elements and attributes are evaluated recursively.)
+    
+      - In Nix expressions, the primop `builtins.toXML` converts a term
+        to an XML representation. This is primarily useful for passing
+        structured information to builders.
+
+  - You can now unambiguously specify which derivation to build or
+    install in `nix-env`, `nix-instantiate` and `nix-build` using the
+    `--attr` / `-A` flags, which takes an attribute name as argument.
+    (Unlike symbolic package names such as `subversion-1.4.0`, attribute
+    names in an attribute set are unique.) For instance, a quick way to
+    perform a test build of a package in Nixpkgs is `nix-build
+            pkgs/top-level/all-packages.nix -A
+            foo`. `nix-env -q
+            --attr` shows the attribute names corresponding to each derivation.
+
+  - If the top-level Nix expression used by `nix-env`, `nix-instantiate`
+    or `nix-build` evaluates to a function whose arguments all have
+    default values, the function will be called automatically. Also, the
+    new command-line switch `--arg
+            name
+            value` can be used to specify function arguments on the command
+    line.
+
+  - `nix-install-package --url
+            URL` allows a package to be installed directly from the given URL.
+
+  - Nix now works behind an HTTP proxy server; just set the standard
+    environment variables http\_proxy, https\_proxy, ftp\_proxy or
+    all\_proxy appropriately. Functions such as `fetchurl` in Nixpkgs
+    also respect these variables.
+
+  - `nix-build -o
+            symlink` allows the symlink to the build result to be named
+    something other than `result`.
+
+  - Platform support:
+    
+      - Support for 64-bit platforms, provided a [suitably patched ATerm
+        library](http://bugzilla.sen.cwi.nl:8080/show_bug.cgi?id=606) is
+        used. Also, files larger than 2 GiB are now supported.
+    
+      - Added support for Cygwin (Windows, `i686-cygwin`), Mac OS X on
+        Intel (`i686-darwin`) and Linux on PowerPC (`powerpc-linux`).
+    
+      - Users of SMP and multicore machines will appreciate that the
+        number of builds to be performed in parallel can now be
+        specified in the configuration file in the `build-max-jobs`
+        setting.
+
+  - Garbage collector improvements:
+    
+      - Open files (such as running programs) are now used as roots of
+        the garbage collector. This prevents programs that have been
+        uninstalled from being garbage collected while they are still
+        running. The script that detects these additional runtime roots
+        (`find-runtime-roots.pl`) is inherently system-specific, but it
+        should work on Linux and on all platforms that have the `lsof`
+        utility.
+    
+      - `nix-store --gc` (a.k.a. `nix-collect-garbage`) prints out the
+        number of bytes freed on standard output. `nix-store
+                                                        --gc --print-dead` shows how many bytes would be freed by an
+        actual garbage collection.
+    
+      - `nix-collect-garbage -d` removes all old generations of *all*
+        profiles before calling the actual garbage collector (`nix-store
+                                                        --gc`). This is an easy way to get rid of all old packages in
+        the Nix store.
+    
+      - `nix-store` now has an operation `--delete` to delete specific
+        paths from the Nix store. It won’t delete reachable
+        (non-garbage) paths unless `--ignore-liveness` is specified.
+
+  - Berkeley DB 4.4’s process registry feature is used to recover from
+    crashed Nix processes.
+
+  - A performance issue has been fixed with the `referer` table, which
+    stores the inverse of the `references` table (i.e., it tells you
+    what store paths refer to a given path). Maintaining this table
+    could take a quadratic amount of time, as well as a quadratic amount
+    of Berkeley DB log file space (in particular when running the
+    garbage collector) (`NIX-23`).
+
+  - Nix now catches the `TERM` and `HUP` signals in addition to the
+    `INT` signal. So you can now do a `killall
+            nix-store` without triggering a database recovery.
+
+  - `bsdiff` updated to version 4.3.
+
+  - Substantial performance improvements in expression evaluation and
+    `nix-env -qa`, all thanks to [Valgrind](http://valgrind.org/).
+    Memory use has been reduced by a factor 8 or so. Big speedup by
+    memoisation of path hashing.
+
+  - Lots of bug fixes, notably:
+    
+      - Make sure that the garbage collector can run successfully when
+        the disk is full (`NIX-18`).
+    
+      - `nix-env` now locks the profile to prevent races between
+        concurrent `nix-env` operations on the same profile (`NIX-7`).
+    
+      - Removed misleading messages from `nix-env -i` (e.g.,
+        ``installing
+                                                        `foo'`` followed by ``uninstalling
+                                                        `foo'``) (`NIX-17`).
+
+  - Nix source distributions are a lot smaller now since we no longer
+    include a full copy of the Berkeley DB source distribution (but only
+    the bits we need).
+
+  - Header files are now installed so that external programs can use the
+    Nix libraries.
diff --git a/doc/manual/src/release-notes/rl-0.11.md b/doc/manual/src/release-notes/rl-0.11.md
new file mode 100644
index 000000000..d2f4d73aa
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.11.md
@@ -0,0 +1,167 @@
+# Release 0.11 (2007-12-31)
+
+Nix 0.11 has many improvements over the previous stable release. The
+most important improvement is secure multi-user support. It also
+features many usability enhancements and language extensions, many of
+them prompted by NixOS, the purely functional Linux distribution based
+on Nix. Here is an (incomplete) list:
+
+  - Secure multi-user support. A single Nix store can now be shared
+    between multiple (possible untrusted) users. This is an important
+    feature for NixOS, where it allows non-root users to install
+    software. The old setuid method for sharing a store between multiple
+    users has been removed. Details for setting up a multi-user store
+    can be found in the manual.
+
+  - The new command `nix-copy-closure` gives you an easy and efficient
+    way to exchange software between machines. It copies the missing
+    parts of the closure of a set of store path to or from a remote
+    machine via `ssh`.
+
+  - A new kind of string literal: strings between double single-quotes
+    (`''`) have indentation “intelligently” removed. This allows large
+    strings (such as shell scripts or configuration file fragments in
+    NixOS) to cleanly follow the indentation of the surrounding
+    expression. It also requires much less escaping, since `''` is less
+    common in most languages than `"`.
+
+  - `nix-env` `--set` modifies the current generation of a profile so
+    that it contains exactly the specified derivation, and nothing else.
+    For example, `nix-env -p /nix/var/nix/profiles/browser --set
+            firefox` lets the profile named `browser` contain just Firefox.
+
+  - `nix-env` now maintains meta-information about installed packages in
+    profiles. The meta-information is the contents of the `meta`
+    attribute of derivations, such as `description` or `homepage`. The
+    command `nix-env -q --xml
+            --meta` shows all meta-information.
+
+  - `nix-env` now uses the `meta.priority` attribute of derivations to
+    resolve filename collisions between packages. Lower priority values
+    denote a higher priority. For instance, the GCC wrapper package and
+    the Binutils package in Nixpkgs both have a file `bin/ld`, so
+    previously if you tried to install both you would get a collision.
+    Now, on the other hand, the GCC wrapper declares a higher priority
+    than Binutils, so the former’s `bin/ld` is symlinked in the user
+    environment.
+
+  - `nix-env -i / -u`: instead of breaking package ties by version,
+    break them by priority and version number. That is, if there are
+    multiple packages with the same name, then pick the package with the
+    highest priority, and only use the version if there are multiple
+    packages with the same priority.
+    
+    This makes it possible to mark specific versions/variant in Nixpkgs
+    more or less desirable than others. A typical example would be a
+    beta version of some package (e.g., `gcc-4.2.0rc1`) which should not
+    be installed even though it is the highest version, except when it
+    is explicitly selected (e.g., `nix-env -i
+            gcc-4.2.0rc1`).
+
+  - `nix-env --set-flag` allows meta attributes of installed packages to
+    be modified. There are several attributes that can be usefully
+    modified, because they affect the behaviour of `nix-env` or the user
+    environment build script:
+    
+      - `meta.priority` can be changed to resolve filename clashes (see
+        above).
+    
+      - `meta.keep` can be set to `true` to prevent the package from
+        being upgraded or replaced. Useful if you want to hang on to an
+        older version of a package.
+    
+      - `meta.active` can be set to `false` to “disable” the package.
+        That is, no symlinks will be generated to the files of the
+        package, but it remains part of the profile (so it won’t be
+        garbage-collected). Set it back to `true` to re-enable the
+        package.
+
+  - `nix-env -q` now has a flag `--prebuilt-only` (`-b`) that causes
+    `nix-env` to show only those derivations whose output is already in
+    the Nix store or that can be substituted (i.e., downloaded from
+    somewhere). In other words, it shows the packages that can be
+    installed “quickly”, i.e., don’t need to be built from source. The
+    `-b` flag is also available in `nix-env -i` and `nix-env -u` to
+    filter out derivations for which no pre-built binary is available.
+
+  - The new option `--argstr` (in `nix-env`, `nix-instantiate` and
+    `nix-build`) is like `--arg`, except that the value is a string. For
+    example, `--argstr system
+            i686-linux` is equivalent to `--arg system
+            \"i686-linux\"` (note that `--argstr` prevents annoying quoting
+    around shell arguments).
+
+  - `nix-store` has a new operation `--read-log` (`-l`) `paths` that
+    shows the build log of the given paths.
+
+  - Nix now uses Berkeley DB 4.5. The database is upgraded
+    automatically, but you should be careful not to use old versions of
+    Nix that still use Berkeley DB 4.4.
+
+  - The option `--max-silent-time` (corresponding to the configuration
+    setting `build-max-silent-time`) allows you to set a timeout on
+    builds — if a build produces no output on `stdout` or `stderr` for
+    the given number of seconds, it is terminated. This is useful for
+    recovering automatically from builds that are stuck in an infinite
+    loop.
+
+  - `nix-channel`: each subscribed channel is its own attribute in the
+    top-level expression generated for the channel. This allows
+    disambiguation (e.g. `nix-env
+            -i -A nixpkgs_unstable.firefox`).
+
+  - The substitutes table has been removed from the database. This makes
+    operations such as `nix-pull` and `nix-channel --update` much, much
+    faster.
+
+  - `nix-pull` now supports bzip2-compressed manifests. This speeds up
+    channels.
+
+  - `nix-prefetch-url` now has a limited form of caching. This is used
+    by `nix-channel` to prevent unnecessary downloads when the channel
+    hasn’t changed.
+
+  - `nix-prefetch-url` now by default computes the SHA-256 hash of the
+    file instead of the MD5 hash. In calls to `fetchurl` you should pass
+    the `sha256` attribute instead of `md5`. You can pass either a
+    hexadecimal or a base-32 encoding of the hash.
+
+  - Nix can now perform builds in an automatically generated “chroot”.
+    This prevents a builder from accessing stuff outside of the Nix
+    store, and thus helps ensure purity. This is an experimental
+    feature.
+
+  - The new command `nix-store
+            --optimise` reduces Nix store disk space usage by finding identical
+    files in the store and hard-linking them to each other. It typically
+    reduces the size of the store by something like 25-35%.
+
+  - `~/.nix-defexpr` can now be a directory, in which case the Nix
+    expressions in that directory are combined into an attribute set,
+    with the file names used as the names of the attributes. The command
+    `nix-env
+            --import` (which set the `~/.nix-defexpr` symlink) is removed.
+
+  - Derivations can specify the new special attribute
+    `allowedReferences` to enforce that the references in the output of
+    a derivation are a subset of a declared set of paths. For example,
+    if `allowedReferences` is an empty list, then the output must not
+    have any references. This is used in NixOS to check that generated
+    files such as initial ramdisks for booting Linux don’t have any
+    dependencies.
+
+  - The new attribute `exportReferencesGraph` allows builders access to
+    the references graph of their inputs. This is used in NixOS for
+    tasks such as generating ISO-9660 images that contain a Nix store
+    populated with the closure of certain paths.
+
+  - Fixed-output derivations (like `fetchurl`) can define the attribute
+    `impureEnvVars` to allow external environment variables to be passed
+    to builders. This is used in Nixpkgs to support proxy configuration,
+    among other things.
+
+  - Several new built-in functions: `builtins.attrNames`,
+    `builtins.filterSource`, `builtins.isAttrs`, `builtins.isFunction`,
+    `builtins.listToAttrs`, `builtins.stringLength`, `builtins.sub`,
+    `builtins.substring`, `throw`, `builtins.trace`,
+    `builtins.readFile`.
diff --git a/doc/manual/src/release-notes/rl-0.12.md b/doc/manual/src/release-notes/rl-0.12.md
new file mode 100644
index 000000000..0dc727b69
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.12.md
@@ -0,0 +1,123 @@
+# Release 0.12 (2008-11-20)
+
+  - Nix no longer uses Berkeley DB to store Nix store metadata. The
+    principal advantages of the new storage scheme are: it works
+    properly over decent implementations of NFS (allowing Nix stores to
+    be shared between multiple machines); no recovery is needed when a
+    Nix process crashes; no write access is needed for read-only
+    operations; no more running out of Berkeley DB locks on certain
+    operations.
+    
+    You still need to compile Nix with Berkeley DB support if you want
+    Nix to automatically convert your old Nix store to the new schema.
+    If you don’t need this, you can build Nix with the `configure`
+    option `--disable-old-db-compat`.
+    
+    After the automatic conversion to the new schema, you can delete the
+    old Berkeley DB files:
+    
+        $ cd /nix/var/nix/db
+        $ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG
+    
+    The new metadata is stored in the directories `/nix/var/nix/db/info`
+    and `/nix/var/nix/db/referrer`. Though the metadata is stored in
+    human-readable plain-text files, they are not intended to be
+    human-editable, as Nix is rather strict about the format.
+    
+    The new storage schema may or may not require less disk space than
+    the Berkeley DB environment, mostly depending on the cluster size of
+    your file system. With 1 KiB clusters (which seems to be the `ext3`
+    default nowadays) it usually takes up much less space.
+
+  - There is a new substituter that copies paths directly from other
+    (remote) Nix stores mounted somewhere in the filesystem. For
+    instance, you can speed up an installation by mounting some remote
+    Nix store that already has the packages in question via NFS or
+    `sshfs`. The environment variable NIX\_OTHER\_STORES specifies the
+    locations of the remote Nix directories, e.g. `/mnt/remote-fs/nix`.
+
+  - New `nix-store` operations `--dump-db` and `--load-db` to dump and
+    reload the Nix database.
+
+  - The garbage collector has a number of new options to allow only some
+    of the garbage to be deleted. The option `--max-freed N` tells the
+    collector to stop after at least N bytes have been deleted. The
+    option `--max-links
+            N` tells it to stop after the link count on `/nix/store` has dropped
+    below N. This is useful for very large Nix stores on filesystems
+    with a 32000 subdirectories limit (like `ext3`). The option
+    `--use-atime` causes store paths to be deleted in order of ascending
+    last access time. This allows non-recently used stuff to be deleted.
+    The option `--max-atime time` specifies an upper limit to the last
+    accessed time of paths that may be deleted. For instance,
+    
+    ``` 
+        $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")
+    ```
+    
+    deletes everything that hasn’t been accessed in two months.
+
+  - `nix-env` now uses optimistic profile locking when performing an
+    operation like installing or upgrading, instead of setting an
+    exclusive lock on the profile. This allows multiple `nix-env -i / -u
+    / -e` operations on the same profile in parallel. If a `nix-env`
+    operation sees at the end that the profile was changed in the
+    meantime by another process, it will just restart. This is generally
+    cheap because the build results are still in the Nix store.
+
+  - The option `--dry-run` is now supported by `nix-store -r` and
+    `nix-build`.
+
+  - The information previously shown by `--dry-run` (i.e., which
+    derivations will be built and which paths will be substituted) is
+    now always shown by `nix-env`, `nix-store -r` and `nix-build`. The
+    total download size of substitutable paths is now also shown. For
+    instance, a build will show something like
+    
+        the following derivations will be built:
+          /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv
+          /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv
+          ...
+        the following paths will be downloaded/copied (30.02 MiB):
+          /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
+          /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
+          ...
+
+  - Language features:
+    
+      - @-patterns as in Haskell. For instance, in a function definition
+        
+            f = args @ {x, y, z}: ...;
+        
+        `args` refers to the argument as a whole, which is further
+        pattern-matched against the attribute set pattern `{x, y, z}`.
+    
+      - “`...`” (ellipsis) patterns. An attribute set pattern can now
+        say `...` at the end of the attribute name list to specify that
+        the function takes *at least* the listed attributes, while
+        ignoring additional attributes. For instance,
+        
+            {stdenv, fetchurl, fuse, ...}: ...
+        
+        defines a function that accepts any attribute set that includes
+        at least the three listed attributes.
+    
+      - New primops: `builtins.parseDrvName` (split a package name
+        string like `"nix-0.12pre12876"` into its name and version
+        components, e.g. `"nix"` and `"0.12pre12876"`),
+        `builtins.compareVersions` (compare two version strings using
+        the same algorithm that `nix-env` uses), `builtins.length`
+        (efficiently compute the length of a list), `builtins.mul`
+        (integer multiplication), `builtins.div` (integer division).
+
+  - `nix-prefetch-url` now supports `mirror://` URLs, provided that the
+    environment variable NIXPKGS\_ALL points at a Nixpkgs tree.
+
+  - Removed the commands `nix-pack-closure` and `nix-unpack-closure`.
+    You can do almost the same thing but much more efficiently by doing
+    `nix-store --export
+            $(nix-store -qR paths) > closure` and `nix-store --import <
+            closure`.
+
+  - Lots of bug fixes, including a big performance bug in the handling
+    of `with`-expressions.
diff --git a/doc/manual/src/release-notes/rl-0.13.md b/doc/manual/src/release-notes/rl-0.13.md
new file mode 100644
index 000000000..21f33e3db
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.13.md
@@ -0,0 +1,55 @@
+# Release 0.13 (2009-11-05)
+
+This is primarily a bug fix release. It has some new features:
+
+  - Syntactic sugar for writing nested attribute sets. Instead of
+    
+        {
+          foo = {
+            bar = 123;
+            xyzzy = true;
+          };
+          a = { b = { c = "d"; }; };
+        }
+    
+    you can write
+    
+        {
+          foo.bar = 123;
+          foo.xyzzy = true;
+          a.b.c = "d";
+        }
+    
+    This is useful, for instance, in NixOS configuration files.
+
+  - Support for Nix channels generated by Hydra, the Nix-based
+    continuous build system. (Hydra generates NAR archives on the fly,
+    so the size and hash of these archives isn’t known in advance.)
+
+  - Support `i686-linux` builds directly on `x86_64-linux` Nix
+    installations. This is implemented using the `personality()`
+    syscall, which causes `uname` to return `i686` in child processes.
+
+  - Various improvements to the `chroot` support. Building in a `chroot`
+    works quite well now.
+
+  - Nix no longer blocks if it tries to build a path and another process
+    is already building the same path. Instead it tries to build another
+    buildable path first. This improves parallelism.
+
+  - Support for large (\> 4 GiB) files in NAR archives.
+
+  - Various (performance) improvements to the remote build mechanism.
+
+  - New primops: `builtins.addErrorContext` (to add a string to stack
+    traces — useful for debugging), `builtins.isBool`,
+    `builtins.isString`, `builtins.isInt`, `builtins.intersectAttrs`.
+
+  - OpenSolaris support (Sander van der Burg).
+
+  - Stack traces are no longer displayed unless the `--show-trace`
+    option is used.
+
+  - The scoping rules for `inherit
+                    (e) ...` in recursive attribute sets have changed. The expression e
+    can now refer to the attributes defined in the containing set.
diff --git a/doc/manual/src/release-notes/rl-0.14.md b/doc/manual/src/release-notes/rl-0.14.md
new file mode 100644
index 000000000..9d58f2072
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.14.md
@@ -0,0 +1,21 @@
+# Release 0.14 (2010-02-04)
+
+This release has the following improvements:
+
+  - The garbage collector now starts deleting garbage much faster than
+    before. It no longer determines liveness of all paths in the store,
+    but does so on demand.
+
+  - Added a new operation, `nix-store --query
+                    --roots`, that shows the garbage collector roots that directly or
+    indirectly point to the given store paths.
+
+  - Removed support for converting Berkeley DB-based Nix databases to
+    the new schema.
+
+  - Removed the `--use-atime` and `--max-atime` garbage collector
+    options. They were not very useful in practice.
+
+  - On Windows, Nix now requires Cygwin 1.7.x.
+
+  - A few bug fixes.
diff --git a/doc/manual/src/release-notes/rl-0.15.md b/doc/manual/src/release-notes/rl-0.15.md
new file mode 100644
index 000000000..48e2a6f1b
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.15.md
@@ -0,0 +1,5 @@
+# Release 0.15 (2010-03-17)
+
+This is a bug-fix release. Among other things, it fixes building on Mac
+OS X (Snow Leopard), and improves the contents of `/etc/passwd` and
+`/etc/group` in `chroot` builds.
diff --git a/doc/manual/src/release-notes/rl-0.16.md b/doc/manual/src/release-notes/rl-0.16.md
new file mode 100644
index 000000000..121bbde28
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.16.md
@@ -0,0 +1,25 @@
+# Release 0.16 (2010-08-17)
+
+This release has the following improvements:
+
+  - The Nix expression evaluator is now much faster in most cases:
+    typically, [3 to 8 times compared to the old
+    implementation](http://www.mail-archive.com/nix-dev@cs.uu.nl/msg04113.html).
+    It also uses less memory. It no longer depends on the ATerm library.
+
+  - Support for configurable parallelism inside builders. Build scripts
+    have always had the ability to perform multiple build actions in
+    parallel (for instance, by running `make -j
+                            2`), but this was not desirable because the number of actions to be
+    performed in parallel was not configurable. Nix now has an option
+    `--cores
+                            N` as well as a configuration setting `build-cores =
+                            N` that causes the environment variable NIX\_BUILD\_CORES to be set
+    to N when the builder is invoked. The builder can use this at its
+    discretion to perform a parallel build, e.g., by calling `make -j
+                            N`. In Nixpkgs, this can be enabled on a per-package basis by
+    setting the derivation attribute `enableParallelBuilding` to `true`.
+
+  - `nix-store -q` now supports XML output through the `--xml` flag.
+
+  - Several bug fixes.
diff --git a/doc/manual/src/release-notes/rl-0.5.md b/doc/manual/src/release-notes/rl-0.5.md
new file mode 100644
index 000000000..5cff2da0b
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.5.md
@@ -0,0 +1,3 @@
+# Release 0.5 and earlier
+
+Please refer to the Subversion commit log messages.
diff --git a/doc/manual/src/release-notes/rl-0.6.md b/doc/manual/src/release-notes/rl-0.6.md
new file mode 100644
index 000000000..c816c9851
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.6.md
@@ -0,0 +1,64 @@
+# Release 0.6 (2004-11-14)
+
+  - Rewrite of the normalisation engine.
+    
+      - Multiple builds can now be performed in parallel (option `-j`).
+    
+      - Distributed builds. Nix can now call a shell script to forward
+        builds to Nix installations on remote machines, which may or may
+        not be of the same platform type.
+    
+      - Option `--fallback` allows recovery from broken substitutes.
+    
+      - Option `--keep-going` causes building of other (unaffected)
+        derivations to continue if one failed.
+
+  - Improvements to the garbage collector (i.e., it should actually work
+    now).
+
+  - Setuid Nix installations allow a Nix store to be shared among
+    multiple users.
+
+  - Substitute registration is much faster now.
+
+  - A utility `nix-build` to build a Nix expression and create a symlink
+    to the result int the current directory; useful for testing Nix
+    derivations.
+
+  - Manual updates.
+
+  - `nix-env` changes:
+    
+      - Derivations for other platforms are filtered out (which can be
+        overridden using `--system-filter`).
+    
+      - `--install` by default now uninstall previous derivations with
+        the same name.
+    
+      - `--upgrade` allows upgrading to a specific version.
+    
+      - New operation `--delete-generations` to remove profile
+        generations (necessary for effective garbage collection).
+    
+      - Nicer output (sorted, columnised).
+
+  - More sensible verbosity levels all around (builder output is now
+    shown always, unless `-Q` is given).
+
+  - Nix expression language changes:
+    
+      - New language construct: `with
+                                                        E1;
+                                                        E2` brings all attributes defined in the attribute set E1 in
+        scope in E2.
+    
+      - Added a `map` function.
+    
+      - Various new operators (e.g., string concatenation).
+
+  - Expression evaluation is much faster.
+
+  - An Emacs mode for editing Nix expressions (with syntax highlighting
+    and indentation) has been added.
+
+  - Many bug fixes.
diff --git a/doc/manual/src/release-notes/rl-0.7.md b/doc/manual/src/release-notes/rl-0.7.md
new file mode 100644
index 000000000..d873fe890
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.7.md
@@ -0,0 +1,21 @@
+# Release 0.7 (2005-01-12)
+
+  - Binary patching. When upgrading components using pre-built binaries
+    (through nix-pull / nix-channel), Nix can automatically download and
+    apply binary patches to already installed components instead of full
+    downloads. Patching is “smart”: if there is a *sequence* of patches
+    to an installed component, Nix will use it. Patches are currently
+    generated automatically between Nixpkgs (pre-)releases.
+
+  - Simplifications to the substitute mechanism.
+
+  - Nix-pull now stores downloaded manifests in
+    `/nix/var/nix/manifests`.
+
+  - Metadata on files in the Nix store is canonicalised after builds:
+    the last-modified timestamp is set to 0 (00:00:00 1/1/1970), the
+    mode is set to 0444 or 0555 (readable and possibly executable by
+    all; setuid/setgid bits are dropped), and the group is set to the
+    default. This ensures that the result of a build and an installation
+    through a substitute is the same; and that timestamp dependencies
+    are revealed.
diff --git a/doc/manual/src/release-notes/rl-0.8.1.md b/doc/manual/src/release-notes/rl-0.8.1.md
new file mode 100644
index 000000000..7629f81cb
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.8.1.md
@@ -0,0 +1,8 @@
+# Release 0.8.1 (2005-04-13)
+
+This is a bug fix release.
+
+  - Patch downloading was broken.
+
+  - The garbage collector would not delete paths that had references
+    from invalid (but substitutable) paths.
diff --git a/doc/manual/src/release-notes/rl-0.8.md b/doc/manual/src/release-notes/rl-0.8.md
new file mode 100644
index 000000000..626c0c92b
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.8.md
@@ -0,0 +1,166 @@
+# Release 0.8 (2005-04-11)
+
+NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a
+result, `nix-pull` manifests and channels built for Nix 0.7 and below
+will not work anymore. However, the Nix expression language has not
+changed, so you can still build from source. Also, existing user
+environments continue to work. Nix 0.8 will automatically upgrade the
+database schema of previous installations when it is first run.
+
+If you get the error message
+
+    you have an old-style manifest `/nix/var/nix/manifests/[...]'; please
+    delete it
+
+you should delete previously downloaded manifests:
+
+    $ rm /nix/var/nix/manifests/*
+
+If `nix-channel` gives the error message
+
+    manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
+    is too old (i.e., for Nix <= 0.7)
+
+then you should unsubscribe from the offending channel (`nix-channel
+--remove
+URL`; leave out `/MANIFEST`), and subscribe to the same URL, with
+`channels` replaced by `channels-v3` (e.g.,
+<http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable>).
+
+Nix 0.8 has the following improvements:
+
+  - The cryptographic hashes used in store paths are now 160 bits long,
+    but encoded in base-32 so that they are still only 32 characters
+    long (e.g.,
+    `/nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0`). (This is
+    actually a 160 bit truncation of a SHA-256 hash.)
+
+  - Big cleanups and simplifications of the basic store semantics. The
+    notion of “closure store expressions” is gone (and so is the notion
+    of “successors”); the file system references of a store path are now
+    just stored in the database.
+    
+    For instance, given any store path, you can query its closure:
+    
+        $ nix-store -qR $(which firefox)
+        ... lots of paths ...
+    
+    Also, Nix now remembers for each store path the derivation that
+    built it (the “deriver”):
+    
+        $ nix-store -qR $(which firefox)
+        /nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv
+    
+    So to see the build-time dependencies, you can do
+    
+        $ nix-store -qR $(nix-store -qd $(which firefox))
+    
+    or, in a nicer format:
+    
+        $ nix-store -q --tree $(nix-store -qd $(which firefox))
+    
+    File system references are also stored in reverse. For instance, you
+    can query all paths that directly or indirectly use a certain Glibc:
+    
+        $ nix-store -q --referrers-closure \
+            /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+
+  - The concept of fixed-output derivations has been formalised.
+    Previously, functions such as `fetchurl` in Nixpkgs used a hack
+    (namely, explicitly specifying a store path hash) to prevent changes
+    to, say, the URL of the file from propagating upwards through the
+    dependency graph, causing rebuilds of everything. This can now be
+    done cleanly by specifying the `outputHash` and `outputHashAlgo`
+    attributes. Nix itself checks that the content of the output has the
+    specified hash. (This is important for maintaining certain
+    invariants necessary for future work on secure shared stores.)
+
+  - One-click installation :-) It is now possible to install any
+    top-level component in Nixpkgs directly, through the web — see,
+    e.g., <http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/>. All you
+    have to do is associate `/nix/bin/nix-install-package` with the MIME
+    type `application/nix-package` (or the extension `.nixpkg`), and
+    clicking on a package link will cause it to be installed, with all
+    appropriate dependencies. If you just want to install some specific
+    application, this is easier than subscribing to a channel.
+
+  - `nix-store -r
+            PATHS` now builds all the derivations PATHS in parallel. Previously
+    it did them sequentially (though exploiting possible parallelism
+    between subderivations). This is nice for build farms.
+
+  - `nix-channel` has new operations `--list` and `--remove`.
+
+  - New ways of installing components into user environments:
+    
+      - Copy from another user environment:
+        
+            $ nix-env -i --from-profile .../other-profile firefox
+    
+      - Install a store derivation directly (bypassing the Nix
+        expression language entirely):
+        
+            $ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv
+        
+        (This is used to implement `nix-install-package`, which is
+        therefore immune to evolution in the Nix expression language.)
+    
+      - Install an already built store path directly:
+        
+            $ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1
+    
+      - Install the result of a Nix expression specified as a
+        command-line argument:
+        
+            $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'
+        
+        The difference with the normal installation mode is that `-E`
+        does not use the `name` attributes of derivations. Therefore,
+        this can be used to disambiguate multiple derivations with the
+        same name.
+
+  - A hash of the contents of a store path is now stored in the database
+    after a successful build. This allows you to check whether store
+    paths have been tampered with: `nix-store
+            --verify --check-contents`.
+
+  - Implemented a concurrent garbage collector. It is now always safe to
+    run the garbage collector, even if other Nix operations are
+    happening simultaneously.
+    
+    However, there can still be GC races if you use `nix-instantiate`
+    and `nix-store
+                    --realise` directly to build things. To prevent races, use the
+    `--add-root` flag of those commands.
+
+  - The garbage collector now finally deletes paths in the right order
+    (i.e., topologically sorted under the “references” relation), thus
+    making it safe to interrupt the collector without risking a store
+    that violates the closure invariant.
+
+  - Likewise, the substitute mechanism now downloads files in the right
+    order, thus preserving the closure invariant at all times.
+
+  - The result of `nix-build` is now registered as a root of the garbage
+    collector. If the `./result` link is deleted, the GC root disappears
+    automatically.
+
+  - The behaviour of the garbage collector can be changed globally by
+    setting options in `/nix/etc/nix/nix.conf`.
+    
+      - `gc-keep-derivations` specifies whether deriver links should be
+        followed when searching for live paths.
+    
+      - `gc-keep-outputs` specifies whether outputs of derivations
+        should be followed when searching for live paths.
+    
+      - `env-keep-derivations` specifies whether user environments
+        should store the paths of derivations when they are added (thus
+        keeping the derivations alive).
+
+  - New `nix-env` query flags `--drv-path` and `--out-path`.
+
+  - `fetchurl` allows SHA-1 and SHA-256 in addition to MD5. Just specify
+    the attribute `sha1` or `sha256` instead of `md5`.
+
+  - Manual updates.
diff --git a/doc/manual/src/release-notes/rl-0.9.1.md b/doc/manual/src/release-notes/rl-0.9.1.md
new file mode 100644
index 000000000..9c20eed1b
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.9.1.md
@@ -0,0 +1,4 @@
+# Release 0.9.1 (2005-09-20)
+
+This bug fix release addresses a problem with the ATerm library when the
+`--with-aterm` flag in `configure` was *not* used.
diff --git a/doc/manual/src/release-notes/rl-0.9.2.md b/doc/manual/src/release-notes/rl-0.9.2.md
new file mode 100644
index 000000000..0caaf28de
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.9.2.md
@@ -0,0 +1,11 @@
+# Release 0.9.2 (2005-09-21)
+
+This bug fix release fixes two problems on Mac OS X:
+
+  - If Nix was linked against statically linked versions of the ATerm or
+    Berkeley DB library, there would be dynamic link errors at runtime.
+
+  - `nix-pull` and `nix-push` intermittently failed due to race
+    conditions involving pipes and child processes with error messages
+    such as `open2: open(GLOB(0x180b2e4), >&=9) failed: Bad
+            file descriptor at /nix/bin/nix-pull line 77` (issue `NIX-14`).
diff --git a/doc/manual/src/release-notes/rl-0.9.md b/doc/manual/src/release-notes/rl-0.9.md
new file mode 100644
index 000000000..8c3e1b28e
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-0.9.md
@@ -0,0 +1,72 @@
+# Release 0.9 (2005-09-16)
+
+NOTE: this version of Nix uses Berkeley DB 4.3 instead of 4.2. The
+database is upgraded automatically, but you should be careful not to use
+old versions of Nix that still use Berkeley DB 4.2. In particular, if
+you use a Nix installed through Nix, you should run
+
+    $ nix-store --clear-substitutes
+
+first.
+
+  - Unpacking of patch sequences is much faster now since we no longer
+    do redundant unpacking and repacking of intermediate paths.
+
+  - Nix now uses Berkeley DB 4.3.
+
+  - The `derivation` primitive is lazier. Attributes of dependent
+    derivations can mutually refer to each other (as long as there are
+    no data dependencies on the `outPath` and `drvPath` attributes
+    computed by `derivation`).
+    
+    For example, the expression `derivation
+            attrs` now evaluates to (essentially)
+    
+        attrs // {
+          type = "derivation";
+          outPath = derivation! attrs;
+          drvPath = derivation! attrs;
+        }
+    
+    where `derivation!` is a primop that does the actual derivation
+    instantiation (i.e., it does what `derivation` used to do). The
+    advantage is that it allows commands such as `nix-env -qa` and
+    `nix-env -i` to be much faster since they no longer need to
+    instantiate all derivations, just the `name` attribute.
+    
+    Also, it allows derivations to cyclically reference each other, for
+    example,
+    
+        webServer = derivation {
+          ...
+          hostName = "svn.cs.uu.nl";
+          services = [svnService];
+        };
+         
+        svnService = derivation {
+          ...
+          hostName = webServer.hostName;
+        };
+    
+    Previously, this would yield a black hole (infinite recursion).
+
+  - `nix-build` now defaults to using `./default.nix` if no Nix
+    expression is specified.
+
+  - `nix-instantiate`, when applied to a Nix expression that evaluates
+    to a function, will call the function automatically if all its
+    arguments have defaults.
+
+  - Nix now uses libtool to build dynamic libraries. This reduces the
+    size of executables.
+
+  - A new list concatenation operator `++`. For example, `[1 2 3] ++
+    [4 5
+            6]` evaluates to `[1 2 3 4 5
+            6]`.
+
+  - Some currently undocumented primops to support low-level build
+    management using Nix (i.e., using Nix as a Make replacement). See
+    the commit messages for `r3578` and `r3580`.
+
+  - Various bug fixes and performance improvements.
diff --git a/doc/manual/src/release-notes/rl-1.0.md b/doc/manual/src/release-notes/rl-1.0.md
new file mode 100644
index 000000000..025f827d9
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.0.md
@@ -0,0 +1,68 @@
+# Release 1.0 (2012-05-11)
+
+There have been numerous improvements and bug fixes since the previous
+release. Here are the most significant:
+
+  - Nix can now optionally use the Boehm garbage collector. This
+    significantly reduces the Nix evaluator’s memory footprint,
+    especially when evaluating large NixOS system configurations. It can
+    be enabled using the `--enable-gc` configure option.
+
+  - Nix now uses SQLite for its database. This is faster and more
+    flexible than the old *ad hoc* format. SQLite is also used to cache
+    the manifests in `/nix/var/nix/manifests`, resulting in a
+    significant speedup.
+
+  - Nix now has an search path for expressions. The search path is set
+    using the environment variable NIX\_PATH and the `-I` command line
+    option. In Nix expressions, paths between angle brackets are used to
+    specify files that must be looked up in the search path. For
+    instance, the expression `<nixpkgs/default.nix>` looks for a file
+    `nixpkgs/default.nix` relative to every element in the search path.
+
+  - The new command `nix-build --run-env` builds all dependencies of a
+    derivation, then starts a shell in an environment containing all
+    variables from the derivation. This is useful for reproducing the
+    environment of a derivation for development.
+
+  - The new command `nix-store --verify-path` verifies that the contents
+    of a store path have not changed.
+
+  - The new command `nix-store --print-env` prints out the environment
+    of a derivation in a format that can be evaluated by a shell.
+
+  - Attribute names can now be arbitrary strings. For instance, you can
+    write `{ "foo-1.2" = …; "bla bla" = …; }."bla
+                    bla"`.
+
+  - Attribute selection can now provide a default value using the `or`
+    operator. For instance, the expression `x.y.z or e` evaluates to the
+    attribute `x.y.z` if it exists, and `e` otherwise.
+
+  - The right-hand side of the `?` operator can now be an attribute
+    path, e.g., `attrs ?
+                    a.b.c`.
+
+  - On Linux, Nix will now make files in the Nix store immutable on
+    filesystems that support it. This prevents accidental modification
+    of files in the store by the root user.
+
+  - Nix has preliminary support for derivations with multiple outputs.
+    This is useful because it allows parts of a package to be deployed
+    and garbage-collected separately. For instance, development parts of
+    a package such as header files or static libraries would typically
+    not be part of the closure of an application, resulting in reduced
+    disk usage and installation time.
+
+  - The Nix store garbage collector is faster and holds the global lock
+    for a shorter amount of time.
+
+  - The option `--timeout` (corresponding to the configuration setting
+    `build-timeout`) allows you to set an absolute timeout on builds —
+    if a build runs for more than the given number of seconds, it is
+    terminated. This is useful for recovering automatically from builds
+    that are stuck in an infinite loop but keep producing output, and
+    for which `--max-silent-time` is ineffective.
+
+  - Nix development has moved to GitHub
+    (<https://github.com/NixOS/nix>).
diff --git a/doc/manual/src/release-notes/rl-1.1.md b/doc/manual/src/release-notes/rl-1.1.md
new file mode 100644
index 000000000..1e658fe15
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.1.md
@@ -0,0 +1,61 @@
+# Release 1.1 (2012-07-18)
+
+This release has the following improvements:
+
+  - On Linux, when doing a chroot build, Nix now uses various namespace
+    features provided by the Linux kernel to improve build isolation.
+    Namely:
+    
+      - The private network namespace ensures that builders cannot talk
+        to the outside world (or vice versa): each build only sees a
+        private loopback interface. This also means that two concurrent
+        builds can listen on the same port (e.g. as part of a test)
+        without conflicting with each other.
+    
+      - The PID namespace causes each build to start as PID 1. Processes
+        outside of the chroot are not visible to those on the inside. On
+        the other hand, processes inside the chroot *are* visible from
+        the outside (though with different PIDs).
+    
+      - The IPC namespace prevents the builder from communicating with
+        outside processes using SysV IPC mechanisms (shared memory,
+        message queues, semaphores). It also ensures that all IPC
+        objects are destroyed when the builder exits.
+    
+      - The UTS namespace ensures that builders see a hostname of
+        `localhost` rather than the actual hostname.
+    
+      - The private mount namespace was already used by Nix to ensure
+        that the bind-mounts used to set up the chroot are cleaned up
+        automatically.
+
+  - Build logs are now compressed using `bzip2`. The command `nix-store
+                    -l` decompresses them on the fly. This can be disabled by setting
+    the option `build-compress-log` to `false`.
+
+  - The creation of build logs in `/nix/var/log/nix/drvs` can be
+    disabled by setting the new option `build-keep-log` to `false`. This
+    is useful, for instance, for Hydra build machines.
+
+  - Nix now reserves some space in `/nix/var/nix/db/reserved` to ensure
+    that the garbage collector can run successfully if the disk is full.
+    This is necessary because SQLite transactions fail if the disk is
+    full.
+
+  - Added a basic `fetchurl` function. This is not intended to replace
+    the `fetchurl` in Nixpkgs, but is useful for bootstrapping; e.g., it
+    will allow us to get rid of the bootstrap binaries in the Nixpkgs
+    source tree and download them instead. You can use it by doing
+    `import <nix/fetchurl.nix> { url =
+                    url; sha256 =
+                    "hash"; }`. (Shea Levy)
+
+  - Improved RPM spec file. (Michel Alexandre Salim)
+
+  - Support for on-demand socket-based activation in the Nix daemon with
+    `systemd`.
+
+  - Added a manpage for nix.conf5.
+
+  - When using the Nix daemon, the `-s` flag in `nix-env -qa` is now
+    much faster.
diff --git a/doc/manual/src/release-notes/rl-1.10.md b/doc/manual/src/release-notes/rl-1.10.md
new file mode 100644
index 000000000..2bb859536
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.10.md
@@ -0,0 +1,31 @@
+# Release 1.10 (2015-09-03)
+
+This is primarily a bug fix release. It also has a number of new
+features:
+
+  - A number of builtin functions have been added to reduce
+    Nixpkgs/NixOS evaluation time and memory consumption: `all`, `any`,
+    `concatStringsSep`, `foldl’`, `genList`, `replaceStrings`, `sort`.
+
+  - The garbage collector is more robust when the disk is full.
+
+  - Nix supports a new API for building derivations that doesn’t require
+    a `.drv` file to be present on disk; it only requires an in-memory
+    representation of the derivation. This is used by the Hydra
+    continuous build system to make remote builds more efficient.
+
+  - The function `<nix/fetchurl.nix>` now uses a *builtin* builder (i.e.
+    it doesn’t require starting an external process; the download is
+    performed by Nix itself). This ensures that derivation paths don’t
+    change when Nix is upgraded, and obviates the need for ugly hacks to
+    support chroot execution.
+
+  - `--version -v` now prints some configuration information, in
+    particular what compile-time optional features are enabled, and the
+    paths of various directories.
+
+  - Build users have their supplementary groups set correctly.
+
+This release has contributions from Eelco Dolstra, Guillaume Maudoux,
+Iwan Aucamp, Jaka Hudoklin, Kirill Elagin, Ludovic Courtès, Manolis
+Ragkousis, Nicolas B. Pierron and Shea Levy.
diff --git a/doc/manual/src/release-notes/rl-1.11.10.md b/doc/manual/src/release-notes/rl-1.11.10.md
new file mode 100644
index 000000000..d1efe1d0b
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.11.10.md
@@ -0,0 +1,21 @@
+# Release 1.11.10 (2017-06-12)
+
+This release fixes a security bug in Nix’s “build user” build isolation
+mechanism. Previously, Nix builders had the ability to create setuid
+binaries owned by a `nixbld` user. Such a binary could then be used by
+an attacker to assume a `nixbld` identity and interfere with subsequent
+builds running under the same UID.
+
+To prevent this issue, Nix now disallows builders to create setuid and
+setgid binaries. On Linux, this is done using a seccomp BPF filter. Note
+that this imposes a small performance penalty (e.g. 1% when building GNU
+Hello). Using seccomp, we now also prevent the creation of extended
+attributes and POSIX ACLs since these cannot be represented in the NAR
+format and (in the case of POSIX ACLs) allow bypassing regular Nix store
+permissions. On macOS, the restriction is implemented using the existing
+sandbox mechanism, which now uses a minimal “allow all except the
+creation of setuid/setgid binaries” profile when regular sandboxing is
+disabled. On other platforms, the “build user” mechanism is now
+disabled.
+
+Thanks go to Linus Heckemann for discovering and reporting this bug.
diff --git a/doc/manual/src/release-notes/rl-1.11.md b/doc/manual/src/release-notes/rl-1.11.md
new file mode 100644
index 000000000..381887bd8
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.11.md
@@ -0,0 +1,87 @@
+# Release 1.11 (2016-01-19)
+
+This is primarily a bug fix release. It also has a number of new
+features:
+
+  - `nix-prefetch-url` can now download URLs specified in a Nix
+    expression. For example,
+    
+        $ nix-prefetch-url -A hello.src
+    
+    will prefetch the file specified by the `fetchurl` call in the
+    attribute `hello.src` from the Nix expression in the current
+    directory, and print the cryptographic hash of the resulting file on
+    stdout. This differs from `nix-build -A
+                    hello.src` in that it doesn't verify the hash, and is thus useful
+    when you’re updating a Nix expression.
+    
+    You can also prefetch the result of functions that unpack a tarball,
+    such as `fetchFromGitHub`. For example:
+    
+        $ nix-prefetch-url --unpack https://github.com/NixOS/patchelf/archive/0.8.tar.gz
+    
+    or from a Nix expression:
+    
+        $ nix-prefetch-url -A nix-repl.src
+
+  - The builtin function `<nix/fetchurl.nix>` now supports downloading
+    and unpacking NARs. This removes the need to have multiple downloads
+    in the Nixpkgs stdenv bootstrap process (like a separate busybox
+    binary for Linux, or curl/mkdir/sh/bzip2 for Darwin). Now all those
+    files can be combined into a single NAR, optionally compressed using
+    `xz`.
+
+  - Nix now supports SHA-512 hashes for verifying fixed-output
+    derivations, and in `builtins.hashString`.
+
+  - The new flag `--option build-repeat
+                            N` will cause every build to be executed N+1 times. If the build
+    output differs between any round, the build is rejected, and the
+    output paths are not registered as valid. This is primarily useful
+    to verify build determinism. (We already had a `--check` option to
+    repeat a previously succeeded build. However, with `--check`,
+    non-deterministic builds are registered in the DB. Preventing that
+    is useful for Hydra to ensure that non-deterministic builds don't
+    end up getting published to the binary cache.)
+
+  - The options `--check` and `--option
+                            build-repeat N`, if they detect a difference between two runs of the
+    same derivation and `-K` is given, will make the output of the other
+    run available under `store-path-check`. This makes it easier to
+    investigate the non-determinism using tools like `diffoscope`, e.g.,
+    
+        $ nix-build pkgs/stdenv/linux -A stage1.pkgs.zlib --check -K
+        error: derivation ‘/nix/store/l54i8wlw2265…-zlib-1.2.8.drv’ may not
+        be deterministic: output ‘/nix/store/11a27shh6n2i…-zlib-1.2.8’
+        differs from ‘/nix/store/11a27shh6n2i…-zlib-1.2.8-check’
+        
+        $ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zlib-1.2.8-check
+        …
+        ├── lib/libz.a
+        │   ├── metadata
+        │   │ @@ -1,15 +1,15 @@
+        │   │ -rw-r--r-- 30001/30000   3096 Jan 12 15:20 2016 adler32.o
+        …
+        │   │ +rw-r--r-- 30001/30000   3096 Jan 12 15:28 2016 adler32.o
+        …
+
+  - Improved FreeBSD support.
+
+  - `nix-env -qa --xml --meta` now prints license information.
+
+  - The maximum number of parallel TCP connections that the binary cache
+    substituter will use has been decreased from 150 to 25. This should
+    prevent upsetting some broken NAT routers, and also improves
+    performance.
+
+  - All "chroot"-containing strings got renamed to "sandbox". In
+    particular, some Nix options got renamed, but the old names are
+    still accepted as lower-priority aliases.
+
+This release has contributions from Anders Claesson, Anthony Cowley,
+Bjørn Forsman, Brian McKenna, Danny Wilson, davidak, Eelco Dolstra,
+Fabian Schmitthenner, FrankHB, Ilya Novoselov, janus, Jim Garrison, John
+Ericson, Jude Taylor, Ludovic Courtès, Manuel Jacob, Mathnerd314, Pascal
+Wittmann, Peter Simons, Philip Potter, Preston Bennes, Rommel M.
+Martinez, Sander van der Burg, Shea Levy, Tim Cuthbertson, Tuomas
+Tynkkynen, Utku Demir and Vladimír Čunát.
diff --git a/doc/manual/src/release-notes/rl-1.2.md b/doc/manual/src/release-notes/rl-1.2.md
new file mode 100644
index 000000000..e62b2dac9
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.2.md
@@ -0,0 +1,97 @@
+# Release 1.2 (2012-12-06)
+
+This release has the following improvements and changes:
+
+  - Nix has a new binary substituter mechanism: the *binary cache*. A
+    binary cache contains pre-built binaries of Nix packages. Whenever
+    Nix wants to build a missing Nix store path, it will check a set of
+    binary caches to see if any of them has a pre-built binary of that
+    path. The configuration setting `binary-caches` contains a list of
+    URLs of binary caches. For instance, doing
+    
+        $ nix-env -i thunderbird --option binary-caches http://cache.nixos.org
+    
+    will install Thunderbird and its dependencies, using the available
+    pre-built binaries in <http://cache.nixos.org>. The main advantage
+    over the old “manifest”-based method of getting pre-built binaries
+    is that you don’t have to worry about your manifest being in sync
+    with the Nix expressions you’re installing from; i.e., you don’t
+    need to run `nix-pull` to update your manifest. It’s also more
+    scalable because you don’t need to redownload a giant manifest file
+    every time.
+    
+    A Nix channel can provide a binary cache URL that will be used
+    automatically if you subscribe to that channel. If you use the
+    Nixpkgs or NixOS channels (<http://nixos.org/channels>) you
+    automatically get the cache <http://cache.nixos.org>.
+    
+    Binary caches are created using `nix-push`. For details on the
+    operation and format of binary caches, see the `nix-push` manpage.
+    More details are provided in [this nix-dev
+    posting](https://nixos.org/nix-dev/2012-September/009826.html).
+
+  - Multiple output support should now be usable. A derivation can
+    declare that it wants to produce multiple store paths by saying
+    something like
+    
+        outputs = [ "lib" "headers" "doc" ];
+    
+    This will cause Nix to pass the intended store path of each output
+    to the builder through the environment variables `lib`, `headers`
+    and `doc`. Other packages can refer to a specific output by
+    referring to `pkg.output`, e.g.
+    
+        buildInputs = [ pkg.lib pkg.headers ];
+    
+    If you install a package with multiple outputs using `nix-env`, each
+    output path will be symlinked into the user environment.
+
+  - Dashes are now valid as part of identifiers and attribute names.
+
+  - The new operation `nix-store --repair-path` allows corrupted or
+    missing store paths to be repaired by redownloading them. `nix-store
+    --verify --check-contents
+                    --repair` will scan and repair all paths in the Nix store.
+    Similarly, `nix-env`, `nix-build`, `nix-instantiate` and `nix-store
+    --realise` have a `--repair` flag to detect and fix bad paths by
+    rebuilding or redownloading them.
+
+  - Nix no longer sets the immutable bit on files in the Nix store.
+    Instead, the recommended way to guard the Nix store against
+    accidental modification on Linux is to make it a read-only bind
+    mount, like this:
+    
+        $ mount --bind /nix/store /nix/store
+        $ mount -o remount,ro,bind /nix/store
+    
+    Nix will automatically make `/nix/store` writable as needed (using a
+    private mount namespace) to allow modifications.
+
+  - Store optimisation (replacing identical files in the store with hard
+    links) can now be done automatically every time a path is added to
+    the store. This is enabled by setting the configuration option
+    `auto-optimise-store` to `true` (disabled by default).
+
+  - Nix now supports `xz` compression for NARs in addition to `bzip2`.
+    It compresses about 30% better on typical archives and decompresses
+    about twice as fast.
+
+  - Basic Nix expression evaluation profiling: setting the environment
+    variable NIX\_COUNT\_CALLS to `1` will cause Nix to print how many
+    times each primop or function was executed.
+
+  - New primops: `concatLists`, `elem`, `elemAt` and `filter`.
+
+  - The command `nix-copy-closure` has a new flag `--use-substitutes`
+    (`-s`) to download missing paths on the target machine using the
+    substitute mechanism.
+
+  - The command `nix-worker` has been renamed to `nix-daemon`. Support
+    for running the Nix worker in “slave” mode has been removed.
+
+  - The `--help` flag of every Nix command now invokes `man`.
+
+  - Chroot builds are now supported on systemd machines.
+
+This release has contributions from Eelco Dolstra, Florian Friesdorf,
+Mats Erik Andersson and Shea Levy.
diff --git a/doc/manual/src/release-notes/rl-1.3.md b/doc/manual/src/release-notes/rl-1.3.md
new file mode 100644
index 000000000..0c7b48380
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.3.md
@@ -0,0 +1,10 @@
+# Release 1.3 (2013-01-04)
+
+This is primarily a bug fix release. When this version is first run on
+Linux, it removes any immutable bits from the Nix store and increases
+the schema version of the Nix store. (The previous release removed
+support for setting the immutable bit; this release clears any remaining
+immutable bits to make certain operations more efficient.)
+
+This release has contributions from Eelco Dolstra and Stuart
+Pernsteiner.
diff --git a/doc/manual/src/release-notes/rl-1.4.md b/doc/manual/src/release-notes/rl-1.4.md
new file mode 100644
index 000000000..d6d2227f8
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.4.md
@@ -0,0 +1,22 @@
+# Release 1.4 (2013-02-26)
+
+This release fixes a security bug in multi-user operation. It was
+possible for derivations to cause the mode of files outside of the Nix
+store to be changed to 444 (read-only but world-readable) by creating
+hard links to those files
+([details](https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a79141ac4)).
+
+There are also the following improvements:
+
+  - New built-in function: `builtins.hashString`.
+
+  - Build logs are now stored in `/nix/var/log/nix/drvs/XX/`, where XX
+    is the first two characters of the derivation. This is useful on
+    machines that keep a lot of build logs (such as Hydra servers).
+
+  - The function `corepkgs/fetchurl` can now make the downloaded file
+    executable. This will allow getting rid of all bootstrap binaries in
+    the Nixpkgs source tree.
+
+  - Language change: The expression `"${./path}
+            ..."` now evaluates to a string instead of a path.
diff --git a/doc/manual/src/release-notes/rl-1.5.1.md b/doc/manual/src/release-notes/rl-1.5.1.md
new file mode 100644
index 000000000..72b29518e
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.5.1.md
@@ -0,0 +1,4 @@
+# Release 1.5.1 (2013-02-28)
+
+The bug fix to the bug fix had a bug itself, of course. But this time it
+will work for sure\!
diff --git a/doc/manual/src/release-notes/rl-1.5.2.md b/doc/manual/src/release-notes/rl-1.5.2.md
new file mode 100644
index 000000000..508580554
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.5.2.md
@@ -0,0 +1,4 @@
+# Release 1.5.2 (2013-05-13)
+
+This is primarily a bug fix release. It has contributions from Eelco
+Dolstra, Lluís Batlle i Rossell and Shea Levy.
diff --git a/doc/manual/src/release-notes/rl-1.5.md b/doc/manual/src/release-notes/rl-1.5.md
new file mode 100644
index 000000000..d2ccf8a5d
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.5.md
@@ -0,0 +1,4 @@
+# Release 1.5 (2013-02-27)
+
+This is a brown paper bag release to fix a regression introduced by the
+hard link security fix in 1.4.
diff --git a/doc/manual/src/release-notes/rl-1.6.1.md b/doc/manual/src/release-notes/rl-1.6.1.md
new file mode 100644
index 000000000..ed974fe0b
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.6.1.md
@@ -0,0 +1,32 @@
+# Release 1.6.1 (2013-10-28)
+
+This is primarily a bug fix release. Changes of interest are:
+
+  - Nix 1.6 accidentally changed the semantics of antiquoted paths in
+    strings, such as `"${/foo}/bar"`. This release reverts to the Nix
+    1.5.3 behaviour.
+
+  - Previously, Nix optimised expressions such as `"${expr}"` to expr.
+    Thus it neither checked whether expr could be coerced to a string,
+    nor applied such coercions. This meant that `"${123}"` evaluatued to
+    `123`, and `"${./foo}"` evaluated to `./foo` (even though `"${./foo}
+    "` evaluates to `"/nix/store/hash-foo "`). Nix now checks the type
+    of antiquoted expressions and applies coercions.
+
+  - Nix now shows the exact position of undefined variables. In
+    particular, undefined variable errors in a `with` previously didn't
+    show *any* position information, so this makes it a lot easier to
+    fix such errors.
+
+  - Undefined variables are now treated consistently. Previously, the
+    `tryEval` function would catch undefined variables inside a `with`
+    but not outside. Now `tryEval` never catches undefined variables.
+
+  - Bash completion in `nix-shell` now works correctly.
+
+  - Stack traces are less verbose: they no longer show calls to builtin
+    functions and only show a single line for each derivation on the
+    call stack.
+
+  - New built-in function: `builtins.typeOf`, which returns the type of
+    its argument as a string.
diff --git a/doc/manual/src/release-notes/rl-1.6.md b/doc/manual/src/release-notes/rl-1.6.md
new file mode 100644
index 000000000..a4583f4e6
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.6.md
@@ -0,0 +1,72 @@
+# Release 1.6 (2013-09-10)
+
+In addition to the usual bug fixes, this release has several new
+features:
+
+  - The command `nix-build --run-env` has been renamed to `nix-shell`.
+
+  - `nix-shell` now sources `$stdenv/setup` *inside* the interactive
+    shell, rather than in a parent shell. This ensures that shell
+    functions defined by `stdenv` can be used in the interactive shell.
+
+  - `nix-shell` has a new flag `--pure` to clear the environment, so you
+    get an environment that more closely corresponds to the “real” Nix
+    build.
+
+  - `nix-shell` now sets the shell prompt (PS1) to ensure that Nix
+    shells are distinguishable from your regular shells.
+
+  - `nix-env` no longer requires a `*` argument to match all packages,
+    so `nix-env -qa` is equivalent to `nix-env
+                    -qa '*'`.
+
+  - `nix-env -i` has a new flag `--remove-all` (`-r`) to remove all
+    previous packages from the profile. This makes it easier to do
+    declarative package management similar to NixOS’s
+    `environment.systemPackages`. For instance, if you have a
+    specification `my-packages.nix` like this:
+    
+        with import <nixpkgs> {};
+        [ thunderbird
+          geeqie
+          ...
+        ]
+    
+    then after any change to this file, you can run:
+    
+        $ nix-env -f my-packages.nix -ir
+    
+    to update your profile to match the specification.
+
+  - The ‘`with`’ language construct is now more lazy. It only evaluates
+    its argument if a variable might actually refer to an attribute in
+    the argument. For instance, this now works:
+    
+        let
+          pkgs = with pkgs; { foo = "old"; bar = foo; } // overrides;
+          overrides = { foo = "new"; };
+        in pkgs.bar
+    
+    This evaluates to `"new"`, while previously it gave an “infinite
+    recursion” error.
+
+  - Nix now has proper integer arithmetic operators. For instance, you
+    can write `x + y` instead of `builtins.add x y`, or `x <
+                    y` instead of `builtins.lessThan x y`. The comparison operators also
+    work on strings.
+
+  - On 64-bit systems, Nix integers are now 64 bits rather than 32 bits.
+
+  - When using the Nix daemon, the `nix-daemon` worker process now runs
+    on the same CPU as the client, on systems that support setting CPU
+    affinity. This gives a significant speedup on some systems.
+
+  - If a stack overflow occurs in the Nix evaluator, you now get a
+    proper error message (rather than “Segmentation fault”) on some
+    systems.
+
+  - In addition to directories, you can now bind-mount regular files in
+    chroots through the (now misnamed) option `build-chroot-dirs`.
+
+This release has contributions from Domen Kožar, Eelco Dolstra, Florian
+Friesdorf, Gergely Risko, Ivan Kozik, Ludovic Courtès and Shea Levy.
diff --git a/doc/manual/src/release-notes/rl-1.7.md b/doc/manual/src/release-notes/rl-1.7.md
new file mode 100644
index 000000000..71a327ed6
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.7.md
@@ -0,0 +1,140 @@
+# Release 1.7 (2014-04-11)
+
+In addition to the usual bug fixes, this release has the following new
+features:
+
+  - Antiquotation is now allowed inside of quoted attribute names (e.g.
+    `set."${foo}"`). In the case where the attribute name is just a
+    single antiquotation, the quotes can be dropped (e.g. the above
+    example can be written `set.${foo}`). If an attribute name inside of
+    a set declaration evaluates to `null` (e.g. `{ ${null} = false; }`),
+    then that attribute is not added to the set.
+
+  - Experimental support for cryptographically signed binary caches. See
+    [the commit for
+    details](https://github.com/NixOS/nix/commit/0fdf4da0e979f992db75cc17376e455ddc5a96d8).
+
+  - An experimental new substituter, `download-via-ssh`, that fetches
+    binaries from remote machines via SSH. Specifying the flags
+    `--option
+                    use-ssh-substituter true --option ssh-substituter-hosts
+                    user@hostname` will cause Nix to download binaries from the
+    specified machine, if it has them.
+
+  - `nix-store -r` and `nix-build` have a new flag, `--check`, that
+    builds a previously built derivation again, and prints an error
+    message if the output is not exactly the same. This helps to verify
+    whether a derivation is truly deterministic. For example:
+    
+        $ nix-build '<nixpkgs>' -A patchelf
+        …
+        $ nix-build '<nixpkgs>' -A patchelf --check
+        …
+        error: derivation `/nix/store/1ipvxs…-patchelf-0.6' may not be deterministic:
+          hash mismatch in output `/nix/store/4pc1dm…-patchelf-0.6.drv'
+
+  - The `nix-instantiate` flags `--eval-only` and `--parse-only` have
+    been renamed to `--eval` and `--parse`, respectively.
+
+  - `nix-instantiate`, `nix-build` and `nix-shell` now have a flag
+    `--expr` (or `-E`) that allows you to specify the expression to be
+    evaluated as a command line argument. For instance, `nix-instantiate
+    --eval -E
+                    '1 + 2'` will print `3`.
+
+  - `nix-shell` improvements:
+    
+      - It has a new flag, `--packages` (or `-p`), that sets up a build
+        environment containing the specified packages from Nixpkgs. For
+        example, the command
+        
+            $ nix-shell -p sqlite xorg.libX11 hello
+        
+        will start a shell in which the given packages are present.
+    
+      - It now uses `shell.nix` as the default expression, falling back
+        to `default.nix` if the former doesn’t exist. This makes it
+        convenient to have a `shell.nix` in your project to set up a
+        nice development environment.
+    
+      - It evaluates the derivation attribute `shellHook`, if set. Since
+        `stdenv` does not normally execute this hook, it allows you to
+        do `nix-shell`-specific setup.
+    
+      - It preserves the user’s timezone setting.
+
+  - In chroots, Nix now sets up a `/dev` containing only a minimal set
+    of devices (such as `/dev/null`). Note that it only does this if you
+    *don’t* have `/dev` listed in your `build-chroot-dirs` setting;
+    otherwise, it will bind-mount the `/dev` from outside the chroot.
+    
+    Similarly, if you don’t have `/dev/pts` listed in
+    `build-chroot-dirs`, Nix will mount a private `devpts` filesystem on
+    the chroot’s `/dev/pts`.
+
+  - New built-in function: `builtins.toJSON`, which returns a JSON
+    representation of a value.
+
+  - `nix-env -q` has a new flag `--json` to print a JSON representation
+    of the installed or available packages.
+
+  - `nix-env` now supports meta attributes with more complex values,
+    such as attribute sets.
+
+  - The `-A` flag now allows attribute names with dots in them, e.g.
+    
+        $ nix-instantiate --eval '<nixos>' -A 'config.systemd.units."nscd.service".text'
+
+  - The `--max-freed` option to `nix-store --gc` now accepts a unit
+    specifier. For example, `nix-store --gc --max-freed
+                    1G` will free up to 1 gigabyte of disk space.
+
+  - `nix-collect-garbage` has a new flag `--delete-older-than` N`d`,
+    which deletes all user environment generations older than N days.
+    Likewise, `nix-env
+                    --delete-generations` accepts a N`d` age limit.
+
+  - Nix now heuristically detects whether a build failure was due to a
+    disk-full condition. In that case, the build is not flagged as
+    “permanently failed”. This is mostly useful for Hydra, which needs
+    to distinguish between permanent and transient build failures.
+
+  - There is a new symbol `__curPos` that expands to an attribute set
+    containing its file name and line and column numbers, e.g. `{ file =
+    "foo.nix"; line = 10;
+                    column = 5; }`. There also is a new builtin function,
+    `unsafeGetAttrPos`, that returns the position of an attribute. This
+    is used by Nixpkgs to provide location information in error
+    messages, e.g.
+    
+        $ nix-build '<nixpkgs>' -A libreoffice --argstr system x86_64-darwin
+        error: the package ‘libreoffice-4.0.5.2’ in ‘.../applications/office/libreoffice/default.nix:263’
+          is not supported on ‘x86_64-darwin’
+
+  - The garbage collector is now more concurrent with other Nix
+    processes because it releases certain locks earlier.
+
+  - The binary tarball installer has been improved. You can now install
+    Nix by running:
+    
+        $ bash <(curl https://nixos.org/nix/install)
+
+  - More evaluation errors include position information. For instance,
+    selecting a missing attribute will print something like
+    
+        error: attribute `nixUnstabl' missing, at /etc/nixos/configurations/misc/eelco/mandark.nix:216:15
+
+  - The command `nix-setuid-helper` is gone.
+
+  - Nix no longer uses Automake, but instead has a non-recursive, GNU
+    Make-based build system.
+
+  - All installed libraries now have the prefix `libnix`. In particular,
+    this gets rid of `libutil`, which could clash with libraries with
+    the same name from other packages.
+
+  - Nix now requires a compiler that supports C++11.
+
+This release has contributions from Danny Wilson, Domen Kožar, Eelco
+Dolstra, Ian-Woo Kim, Ludovic Courtès, Maxim Ivanov, Petr Rockai,
+Ricardo M. Correia and Shea Levy.
diff --git a/doc/manual/src/release-notes/rl-1.8.md b/doc/manual/src/release-notes/rl-1.8.md
new file mode 100644
index 000000000..10a82d52a
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.8.md
@@ -0,0 +1,88 @@
+# Release 1.8 (2014-12-14)
+
+  - Breaking change: to address a race condition, the remote build hook
+    mechanism now uses `nix-store
+            --serve` on the remote machine. This requires build slaves to be
+    updated to Nix 1.8.
+
+  - Nix now uses HTTPS instead of HTTP to access the default binary
+    cache, `cache.nixos.org`.
+
+  - `nix-env` selectors are now regular expressions. For instance, you
+    can do
+    
+        $ nix-env -qa '.*zip.*'
+    
+    to query all packages with a name containing `zip`.
+
+  - `nix-store --read-log` can now fetch remote build logs. If a build
+    log is not available locally, then ‘nix-store -l’ will now try to
+    download it from the servers listed in the ‘log-servers’ option in
+    nix.conf. For instance, if you have the configuration option
+    
+        log-servers = http://hydra.nixos.org/log
+    
+    then it will try to get logs from `http://hydra.nixos.org/log/base
+    name of the
+    store path`. This allows you to do things like:
+    
+        $ nix-store -l $(which xterm)
+    
+    and get a log even if `xterm` wasn't built locally.
+
+  - New builtin functions: `attrValues`, `deepSeq`, `fromJSON`,
+    `readDir`, `seq`.
+
+  - `nix-instantiate --eval` now has a `--json` flag to print the
+    resulting value in JSON format.
+
+  - `nix-copy-closure` now uses `nix-store --serve` on the remote side
+    to send or receive closures. This fixes a race condition between
+    `nix-copy-closure` and the garbage collector.
+
+  - Derivations can specify the new special attribute
+    `allowedRequisites`, which has a similar meaning to
+    `allowedReferences`. But instead of only enforcing to explicitly
+    specify the immediate references, it requires the derivation to
+    specify all the dependencies recursively (hence the name,
+    requisites) that are used by the resulting output.
+
+  - On Mac OS X, Nix now handles case collisions when importing closures
+    from case-sensitive file systems. This is mostly useful for running
+    NixOps on Mac OS X.
+
+  - The Nix daemon has new configuration options `allowed-users`
+    (specifying the users and groups that are allowed to connect to the
+    daemon) and `trusted-users` (specifying the users and groups that
+    can perform privileged operations like specifying untrusted binary
+    caches).
+
+  - The configuration option `build-cores` now defaults to the number of
+    available CPU cores.
+
+  - Build users are now used by default when Nix is invoked as root.
+    This prevents builds from accidentally running as root.
+
+  - Nix now includes systemd units and Upstart jobs.
+
+  - Speed improvements to `nix-store
+            --optimise`.
+
+  - Language change: the `==` operator now ignores string contexts (the
+    “dependencies” of a string).
+
+  - Nix now filters out Nix-specific ANSI escape sequences on standard
+    error. They are supposed to be invisible, but some terminals show
+    them anyway.
+
+  - Various commands now automatically pipe their output into the pager
+    as specified by the PAGER environment variable.
+
+  - Several improvements to reduce memory consumption in the evaluator.
+
+This release has contributions from Adam Szkoda, Aristid Breitkreuz, Bob
+van der Linden, Charles Strahan, darealshinji, Eelco Dolstra, Gergely
+Risko, Joel Taylor, Ludovic Courtès, Marko Durkovic, Mikey Ariel, Paul
+Colomiets, Ricardo M. Correia, Ricky Elrod, Robert Helgesson, Rob
+Vermaas, Russell O'Connor, Shea Levy, Shell Turner, Sönke Hahn, Steve
+Purcell, Vladimír Čunát and Wout Mertens.
diff --git a/doc/manual/src/release-notes/rl-1.9.md b/doc/manual/src/release-notes/rl-1.9.md
new file mode 100644
index 000000000..01b067aab
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-1.9.md
@@ -0,0 +1,143 @@
+# Release 1.9 (2015-06-12)
+
+In addition to the usual bug fixes, this release has the following new
+features:
+
+  - Signed binary cache support. You can enable signature checking by
+    adding the following to `nix.conf`:
+    
+        signed-binary-caches = *
+        binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+    
+    This will prevent Nix from downloading any binary from the cache
+    that is not signed by one of the keys listed in
+    `binary-cache-public-keys`.
+    
+    Signature checking is only supported if you built Nix with the
+    `libsodium` package.
+    
+    Note that while Nix has had experimental support for signed binary
+    caches since version 1.7, this release changes the signature format
+    in a backwards-incompatible way.
+
+  - Automatic downloading of Nix expression tarballs. In various places,
+    you can now specify the URL of a tarball containing Nix expressions
+    (such as Nixpkgs), which will be downloaded and unpacked
+    automatically. For example:
+    
+      - In `nix-env`:
+        
+            $ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+        
+        This installs Firefox from the latest tested and built revision
+        of the NixOS 14.12 channel.
+    
+      - In `nix-build` and `nix-shell`:
+        
+            $ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
+        
+        This builds GNU Hello from the latest revision of the Nixpkgs
+        master branch.
+    
+      - In the Nix search path (as specified via NIX\_PATH or `-I`). For
+        example, to start a shell containing the Pan package from a
+        specific version of Nixpkgs:
+        
+            $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+    
+      - In `nixos-rebuild` (on NixOS):
+        
+            $ nixos-rebuild test -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz
+    
+      - In Nix expressions, via the new builtin function `fetchTarball`:
+        
+            with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; …
+        
+        (This is not allowed in restricted mode.)
+
+  - `nix-shell` improvements:
+    
+      - `nix-shell` now has a flag `--run` to execute a command in the
+        `nix-shell` environment, e.g. `nix-shell --run make`. This is
+        like the existing `--command` flag, except that it uses a
+        non-interactive shell (ensuring that hitting Ctrl-C won’t drop
+        you into the child shell).
+    
+      - `nix-shell` can now be used as a `#!`-interpreter. This allows
+        you to write scripts that dynamically fetch their own
+        dependencies. For example, here is a Haskell script that, when
+        invoked, first downloads GHC and the Haskell packages on which
+        it depends:
+        
+            #! /usr/bin/env nix-shell
+            #! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP
+            
+            import Network.HTTP
+            
+            main = do
+              resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
+              body <- getResponseBody resp
+              print (take 100 body)
+        
+        Of course, the dependencies are cached in the Nix store, so the
+        second invocation of this script will be much faster.
+
+  - Chroot improvements:
+    
+      - Chroot builds are now supported on Mac OS X (using its sandbox
+        mechanism).
+    
+      - If chroots are enabled, they are now used for all derivations,
+        including fixed-output derivations (such as `fetchurl`). The
+        latter do have network access, but can no longer access the host
+        filesystem. If you need the old behaviour, you can set the
+        option `build-use-chroot` to `relaxed`.
+    
+      - On Linux, if chroots are enabled, builds are performed in a
+        private PID namespace once again. (This functionality was lost
+        in Nix 1.8.)
+    
+      - Store paths listed in `build-chroot-dirs` are now automatically
+        expanded to their closure. For instance, if you want
+        `/nix/store/…-bash/bin/sh` mounted in your chroot as `/bin/sh`,
+        you only need to say `build-chroot-dirs =
+                                                        /bin/sh=/nix/store/…-bash/bin/sh`; it is no longer necessary to
+        specify the dependencies of Bash.
+
+  - The new derivation attribute `passAsFile` allows you to specify that
+    the contents of derivation attributes should be passed via files
+    rather than environment variables. This is useful if you need to
+    pass very long strings that exceed the size limit of the
+    environment. The Nixpkgs function `writeTextFile` uses this.
+
+  - You can now use `~` in Nix file names to refer to your home
+    directory, e.g. `import
+            ~/.nixpkgs/config.nix`.
+
+  - Nix has a new option `restrict-eval` that allows limiting what paths
+    the Nix evaluator has access to. By passing `--option restrict-eval
+    true` to Nix, the evaluator will throw an exception if an attempt is
+    made to access any file outside of the Nix search path. This is
+    primarily intended for Hydra to ensure that a Hydra jobset only
+    refers to its declared inputs (and is therefore reproducible).
+
+  - `nix-env` now only creates a new “generation” symlink in
+    `/nix/var/nix/profiles` if something actually changed.
+
+  - The environment variable NIX\_PAGER can now be set to override
+    PAGER. You can set it to `cat` to disable paging for Nix commands
+    only.
+
+  - Failing `<...>` lookups now show position information.
+
+  - Improved Boehm GC use: we disabled scanning for interior pointers,
+    which should reduce the “`Repeated
+            allocation of very large block`” warnings and associated retention
+    of memory.
+
+This release has contributions from aszlig, Benjamin Staffin, Charles
+Strahan, Christian Theune, Daniel Hahler, Danylo Hlynskyi Daniel
+Peebles, Dan Peebles, Domen Kožar, Eelco Dolstra, Harald van Dijk, Hoang
+Xuan Phu, Jaka Hudoklin, Jeff Ramnani, j-keck, Linquize, Luca Bruno,
+Michael Merickel, Oliver Dunkl, Rob Vermaas, Rok Garbas, Shea Levy,
+Tobias Geerinckx-Rice and William A. Kennington III.
diff --git a/doc/manual/src/release-notes/rl-2.0.md b/doc/manual/src/release-notes/rl-2.0.md
new file mode 100644
index 000000000..0ce985b2f
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.0.md
@@ -0,0 +1,557 @@
+# Release 2.0 (2018-02-22)
+
+The following incompatible changes have been made:
+
+  - The manifest-based substituter mechanism
+    (`download-using-manifests`) has been
+    [removed](https://github.com/NixOS/nix/commit/867967265b80946dfe1db72d40324b4f9af988ed).
+    It has been superseded by the binary cache substituter mechanism
+    since several years. As a result, the following programs have been
+    removed:
+    
+      - `nix-pull`
+    
+      - `nix-generate-patches`
+    
+      - `bsdiff`
+    
+      - `bspatch`
+
+  - The “copy from other stores” substituter mechanism
+    (`copy-from-other-stores` and the NIX\_OTHER\_STORES environment
+    variable) has been removed. It was primarily used by the NixOS
+    installer to copy available paths from the installation medium. The
+    replacement is to use a chroot store as a substituter (e.g.
+    `--substituters /mnt`), or to build into a chroot store (e.g.
+    `--store /mnt --substituters /`).
+
+  - The command `nix-push` has been removed as part of the effort to
+    eliminate Nix's dependency on Perl. You can use `nix copy` instead,
+    e.g. `nix copy
+                    --to file:///tmp/my-binary-cache paths…`
+
+  - The “nested” log output feature (`--log-type
+                    pretty`) has been removed. As a result, `nix-log2xml` was also
+    removed.
+
+  - OpenSSL-based signing has been
+    [removed](https://github.com/NixOS/nix/commit/f435f8247553656774dd1b2c88e9de5d59cab203).
+    This feature was never well-supported. A better alternative is
+    provided by the `secret-key-files` and `trusted-public-keys`
+    options.
+
+  - Failed build caching has been
+    [removed](https://github.com/NixOS/nix/commit/8cffec84859cec8b610a2a22ab0c4d462a9351ff).
+    This feature was introduced to support the Hydra continuous build
+    system, but Hydra no longer uses it.
+
+  - `nix-mode.el` has been removed from Nix. It is now [a separate
+    repository](https://github.com/NixOS/nix-mode) and can be installed
+    through the MELPA package repository.
+
+This release has the following new features:
+
+  - It introduces a new command named `nix`, which is intended to
+    eventually replace all `nix-*` commands with a more consistent and
+    better designed user interface. It currently provides replacements
+    for some (but not all) of the functionality provided by `nix-store`,
+    `nix-build`, `nix-shell -p`, `nix-env -qa`, `nix-instantiate
+    --eval`, `nix-push` and `nix-copy-closure`. It has the following
+    major features:
+    
+      - Unlike the legacy commands, it has a consistent way to refer to
+        packages and package-like arguments (like store paths). For
+        example, the following commands all copy the GNU Hello package
+        to a remote machine:
+        
+            nix copy --to ssh://machine nixpkgs.hello
+        
+            nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
+        
+            nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)'
+        
+        By contrast, `nix-copy-closure` only accepted store paths as
+        arguments.
+    
+      - It is self-documenting: `--help` shows all available
+        command-line arguments. If `--help` is given after a subcommand,
+        it shows examples for that subcommand. `nix
+                                                                        --help-config` shows all configuration options.
+    
+      - It is much less verbose. By default, it displays a single-line
+        progress indicator that shows how many packages are left to be
+        built or downloaded, and (if there are running builds) the most
+        recent line of builder output. If a build fails, it shows the
+        last few lines of builder output. The full build log can be
+        retrieved using `nix
+                                                                        log`.
+    
+      - It
+        [provides](https://github.com/NixOS/nix/commit/b8283773bd64d7da6859ed520ee19867742a03ba)
+        all `nix.conf` configuration options as command line flags. For
+        example, instead of `--option
+                                                                        http-connections 100` you can write `--http-connections 100`.
+        Boolean options can be written as `--foo` or `--no-foo` (e.g.
+        `--no-auto-optimise-store`).
+    
+      - Many subcommands have a `--json` flag to write results to stdout
+        in JSON format.
+    
+    > **Warning**
+    > 
+    > Please note that the `nix` command is a work in progress and the
+    > interface is subject to change.
+    
+    It provides the following high-level (“porcelain”) subcommands:
+    
+      - `nix build` is a replacement for `nix-build`.
+    
+      - `nix run` executes a command in an environment in which the
+        specified packages are available. It is (roughly) a replacement
+        for `nix-shell
+                                                                        -p`. Unlike that command, it does not execute the command in a
+        shell, and has a flag (`-c`) that specifies the unquoted command
+        line to be executed.
+        
+        It is particularly useful in conjunction with chroot stores,
+        allowing Linux users who do not have permission to install Nix
+        in `/nix/store` to still use binary substitutes that assume
+        `/nix/store`. For example,
+        
+            nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!'
+        
+        downloads (or if not substitutes are available, builds) the GNU
+        Hello package into `~/my-nix/nix/store`, then runs `hello` in a
+        mount namespace where `~/my-nix/nix/store` is mounted onto
+        `/nix/store`.
+    
+      - `nix search` replaces `nix-env
+                                                                        -qa`. It searches the available packages for occurrences of a
+        search string in the attribute name, package name or
+        description. Unlike `nix-env -qa`, it has a cache to speed up
+        subsequent searches.
+    
+      - `nix copy` copies paths between arbitrary Nix stores,
+        generalising `nix-copy-closure` and `nix-push`.
+    
+      - `nix repl` replaces the external program `nix-repl`. It provides
+        an interactive environment for evaluating and building Nix
+        expressions. Note that it uses `linenoise-ng` instead of GNU
+        Readline.
+    
+      - `nix upgrade-nix` upgrades Nix to the latest stable version.
+        This requires that Nix is installed in a profile. (Thus it won’t
+        work on NixOS, or if it’s installed outside of the Nix store.)
+    
+      - `nix verify` checks whether store paths are unmodified and/or
+        “trusted” (see below). It replaces `nix-store --verify` and
+        `nix-store
+                                                                        --verify-path`.
+    
+      - `nix log` shows the build log of a package or path. If the build
+        log is not available locally, it will try to obtain it from the
+        configured substituters (such as
+        [cache.nixos.org](cache.nixos.org), which now provides build
+        logs).
+    
+      - `nix edit` opens the source code of a package in your editor.
+    
+      - `nix eval` replaces `nix-instantiate --eval`.
+    
+      - `nix
+                                                                        why-depends` shows why one store path has another in its
+        closure. This is primarily useful to finding the causes of
+        closure bloat. For example,
+        
+            nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev
+        
+        shows a chain of files and fragments of file contents that cause
+        the VLC package to have the “dev” output of `libdrm` in its
+        closure — an undesirable situation.
+    
+      - `nix path-info` shows information about store paths, replacing
+        `nix-store -q`. A useful feature is the option `--closure-size`
+        (`-S`). For example, the following command show the closure
+        sizes of every path in the current NixOS system closure, sorted
+        by size:
+        
+            nix path-info -rS /run/current-system | sort -nk2
+    
+      - `nix optimise-store` replaces `nix-store --optimise`. The main
+        difference is that it has a progress indicator.
+    
+    A number of low-level (“plumbing”) commands are also available:
+    
+      - `nix ls-store` and `nix
+                                                                        ls-nar` list the contents of a store path or NAR file. The
+        former is primarily useful in conjunction with remote stores,
+        e.g.
+        
+            nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
+        
+        lists the contents of path in a binary cache.
+    
+      - `nix cat-store` and `nix
+                                                                        cat-nar` allow extracting a file from a store path or NAR file.
+    
+      - `nix dump-path` writes the contents of a store path to stdout in
+        NAR format. This replaces `nix-store --dump`.
+    
+      - `nix
+                                                                        show-derivation` displays a store derivation in JSON format.
+        This is an alternative to `pp-aterm`.
+    
+      - `nix
+                                                                        add-to-store` replaces `nix-store
+                                                                        --add`.
+    
+      - `nix sign-paths` signs store paths.
+    
+      - `nix copy-sigs` copies signatures from one store to another.
+    
+      - `nix show-config` shows all configuration options and their
+        current values.
+
+  - The store abstraction that Nix has had for a long time to support
+    store access via the Nix daemon has been extended significantly. In
+    particular, substituters (which used to be external programs such as
+    `download-from-binary-cache`) are now subclasses of the abstract
+    `Store` class. This allows many Nix commands to operate on such
+    store types. For example, `nix path-info` shows information about
+    paths in your local Nix store, while `nix path-info --store
+                    https://cache.nixos.org/` shows information about paths in the
+    specified binary cache. Similarly, `nix-copy-closure`, `nix-push`
+    and substitution are all instances of the general notion of copying
+    paths between different kinds of Nix stores.
+    
+    Stores are specified using an URI-like syntax, e.g.
+    <https://cache.nixos.org/> or <ssh://machine>. The following store
+    types are supported:
+    
+      - `LocalStore` (stori URI `local` or an absolute path) and the
+        misnamed `RemoteStore` (`daemon`) provide access to a local Nix
+        store, the latter via the Nix daemon. You can use `auto` or the
+        empty string to auto-select a local or daemon store depending on
+        whether you have write permission to the Nix store. It is no
+        longer necessary to set the NIX\_REMOTE environment variable to
+        use the Nix daemon.
+        
+        As noted above, `LocalStore` now supports chroot builds,
+        allowing the “physical” location of the Nix store (e.g.
+        `/home/alice/nix/store`) to differ from its “logical” location
+        (typically `/nix/store`). This allows non-root users to use Nix
+        while still getting the benefits from prebuilt binaries from
+        [cache.nixos.org](cache.nixos.org).
+    
+      - `BinaryCacheStore` is the abstract superclass of all binary
+        cache stores. It supports writing build logs and NAR content
+        listings in JSON format.
+    
+      - `HttpBinaryCacheStore` (`http://`, `https://`) supports binary
+        caches via HTTP or HTTPS. If the server supports `PUT` requests,
+        it supports uploading store paths via commands such as `nix
+                                                                        copy`.
+    
+      - `LocalBinaryCacheStore` (`file://`) supports binary caches in
+        the local filesystem.
+    
+      - `S3BinaryCacheStore` (`s3://`) supports binary caches stored in
+        Amazon S3, if enabled at compile time.
+    
+      - `LegacySSHStore` (`ssh://`) is used to implement remote builds
+        and `nix-copy-closure`.
+    
+      - `SSHStore` (`ssh-ng://`) supports arbitrary Nix operations on a
+        remote machine via the same protocol used by `nix-daemon`.
+
+  - Security has been improved in various ways:
+    
+      - Nix now stores signatures for local store paths. When paths are
+        copied between stores (e.g., copied from a binary cache to a
+        local store), signatures are propagated.
+        
+        Locally-built paths are signed automatically using the secret
+        keys specified by the `secret-key-files` store option.
+        Secret/public key pairs can be generated using `nix-store
+                                                                        --generate-binary-cache-key`.
+        
+        In addition, locally-built store paths are marked as “ultimately
+        trusted”, but this bit is not propagated when paths are copied
+        between stores.
+    
+      - Content-addressable store paths no longer require signatures —
+        they can be imported into a store by unprivileged users even if
+        they lack signatures.
+    
+      - The command `nix verify` checks whether the specified paths are
+        trusted, i.e., have a certain number of trusted signatures, are
+        ultimately trusted, or are content-addressed.
+    
+      - Substitutions from binary caches
+        [now](https://github.com/NixOS/nix/commit/ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b)
+        require signatures by default. This was already the case on
+        NixOS.
+    
+      - In Linux sandbox builds, we
+        [now](https://github.com/NixOS/nix/commit/eba840c8a13b465ace90172ff76a0db2899ab11b)
+        use `/build` instead of `/tmp` as the temporary build directory.
+        This fixes potential security problems when a build accidentally
+        stores its TMPDIR in some security-sensitive place, such as an
+        RPATH.
+
+  - *Pure evaluation mode*. With the `--pure-eval` flag, Nix enables a
+    variant of the existing restricted evaluation mode that forbids
+    access to anything that could cause different evaluations of the
+    same command line arguments to produce a different result. This
+    includes builtin functions such as `builtins.getEnv`, but more
+    importantly, *all* filesystem or network access unless a content
+    hash or commit hash is specified. For example, calls to
+    `builtins.fetchGit` are only allowed if a `rev` attribute is
+    specified.
+    
+    The goal of this feature is to enable true reproducibility and
+    traceability of builds (including NixOS system configurations) at
+    the evaluation level. For example, in the future, `nixos-rebuild`
+    might build configurations from a Nix expression in a Git repository
+    in pure mode. That expression might fetch other repositories such as
+    Nixpkgs via `builtins.fetchGit`. The commit hash of the top-level
+    repository then uniquely identifies a running system, and, in
+    conjunction with that repository, allows it to be reproduced or
+    modified.
+
+  - There are several new features to support binary reproducibility
+    (i.e. to help ensure that multiple builds of the same derivation
+    produce exactly the same output). When `enforce-determinism` is set
+    to `false`, it’s [no
+    longer](https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051)
+    a fatal error if build rounds produce different output. Also, a hook
+    named `diff-hook` is
+    [provided](https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169)
+    to allow you to run tools such as `diffoscope` when build rounds
+    produce different output.
+
+  - Configuring remote builds is a lot easier now. Provided you are not
+    using the Nix daemon, you can now just specify a remote build
+    machine on the command line, e.g. `--option builders
+                    'ssh://my-mac x86_64-darwin'`. The environment variable
+    NIX\_BUILD\_HOOK has been removed and is no longer needed. The
+    environment variable NIX\_REMOTE\_SYSTEMS is still supported for
+    compatibility, but it is also possible to specify builders in
+    `nix.conf` by setting the option `builders =
+                    @path`.
+
+  - If a fixed-output derivation produces a result with an incorrect
+    hash, the output path is moved to the location corresponding to the
+    actual hash and registered as valid. Thus, a subsequent build of the
+    fixed-output derivation with the correct hash is unnecessary.
+
+  - `nix-shell`
+    [now](https://github.com/NixOS/nix/commit/ea59f39326c8e9dc42dfed4bcbf597fbce58797c)
+    sets the `IN_NIX_SHELL` environment variable during evaluation and
+    in the shell itself. This can be used to perform different actions
+    depending on whether you’re in a Nix shell or in a regular build.
+    Nixpkgs provides `lib.inNixShell` to check this variable during
+    evaluation.
+
+  - NIX\_PATH is now lazy, so URIs in the path are only downloaded if
+    they are needed for evaluation.
+
+  - You can now use <channel:> as a short-hand for
+    <https://nixos.org/channels//nixexprs.tar.xz>. For example,
+    `nix-build channel:nixos-15.09 -A hello` will build the GNU Hello
+    package from the `nixos-15.09` channel. In the future, this may use
+    Git to fetch updates more efficiently.
+
+  - When `--no-build-output` is given, the last 10 lines of the build
+    log will be shown if a build fails.
+
+  - Networking has been improved:
+    
+      - HTTP/2 is now supported. This makes binary cache lookups [much
+        more
+        efficient](https://github.com/NixOS/nix/commit/90ad02bf626b885a5dd8967894e2eafc953bdf92).
+    
+      - We now retry downloads on many HTTP errors, making binary caches
+        substituters more resilient to temporary failures.
+    
+      - HTTP credentials can now be configured via the standard `netrc`
+        mechanism.
+    
+      - If S3 support is enabled at compile time, <s3://> URIs are
+        [supported](https://github.com/NixOS/nix/commit/9ff9c3f2f80ba4108e9c945bbfda2c64735f987b)
+        in all places where Nix allows URIs.
+    
+      - Brotli compression is now supported. In particular,
+        [cache.nixos.org](cache.nixos.org) build logs are now compressed
+        using Brotli.
+
+  - `nix-env`
+    [now](https://github.com/NixOS/nix/commit/b0cb11722626e906a73f10dd9a0c9eea29faf43a)
+    ignores packages with bad derivation names (in particular those
+    starting with a digit or containing a dot).
+
+  - Many configuration options have been renamed, either because they
+    were unnecessarily verbose (e.g. `build-use-sandbox` is now just
+    `sandbox`) or to reflect generalised behaviour (e.g. `binary-caches`
+    is now `substituters` because it allows arbitrary store URIs). The
+    old names are still supported for compatibility.
+
+  - The `max-jobs` option can
+    [now](https://github.com/NixOS/nix/commit/7251d048fa812d2551b7003bc9f13a8f5d4c95a5)
+    be set to `auto` to use the number of CPUs in the system.
+
+  - Hashes can
+    [now](https://github.com/NixOS/nix/commit/c0015e87af70f539f24d2aa2bc224a9d8b84276b)
+    be specified in base-64 format, in addition to base-16 and the
+    non-standard base-32.
+
+  - `nix-shell` now uses `bashInteractive` from Nixpkgs, rather than the
+    `bash` command that happens to be in the caller’s PATH. This is
+    especially important on macOS where the `bash` provided by the
+    system is seriously outdated and cannot execute `stdenv`’s setup
+    script.
+
+  - Nix can now automatically trigger a garbage collection if free disk
+    space drops below a certain level during a build. This is configured
+    using the `min-free` and `max-free` options.
+
+  - `nix-store -q --roots` and `nix-store --gc --print-roots` now show
+    temporary and in-memory roots.
+
+  - Nix can now be extended with plugins. See the documentation of the
+    `plugin-files` option for more details.
+
+The Nix language has the following new features:
+
+  - It supports floating point numbers. They are based on the C++
+    `float` type and are supported by the existing numerical operators.
+    Export and import to and from JSON and XML works, too.
+
+  - Derivation attributes can now reference the outputs of the
+    derivation using the `placeholder` builtin function. For example,
+    the attribute
+    
+        configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";
+    
+    will cause the configureFlags environment variable to contain the
+    actual store paths corresponding to the `out` and `dev` outputs.
+
+The following builtin functions are new or extended:
+
+  - `builtins.fetchGit` allows Git repositories to be fetched at
+    evaluation time. Thus it differs from the `fetchgit` function in
+    Nixpkgs, which fetches at build time and cannot be used to fetch Nix
+    expressions during evaluation. A typical use case is to import
+    external NixOS modules from your configuration, e.g.
+    
+        imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ];
+
+  - Similarly, `builtins.fetchMercurial` allows you to fetch Mercurial
+    repositories.
+
+  - `builtins.path` generalises `builtins.filterSource` and path
+    literals (e.g. `./foo`). It allows specifying a store path name that
+    differs from the source path name (e.g. `builtins.path { path =
+    ./foo; name = "bar";
+                    }`) and also supports filtering out unwanted files.
+
+  - `builtins.fetchurl` and `builtins.fetchTarball` now support `sha256`
+    and `name` attributes.
+
+  - `builtins.split` splits a string using a POSIX extended regular
+    expression as the separator.
+
+  - `builtins.partition` partitions the elements of a list into two
+    lists, depending on a Boolean predicate.
+
+  - `<nix/fetchurl.nix>` now uses the content-addressable tarball cache
+    at <http://tarballs.nixos.org/>, just like `fetchurl` in Nixpkgs.
+    (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)
+
+  - In restricted and pure evaluation mode, builtin functions that
+    download from the network (such as `fetchGit`) are permitted to
+    fetch underneath a list of URI prefixes specified in the option
+    `allowed-uris`.
+
+The Nix build environment has the following changes:
+
+  - Values such as Booleans, integers, (nested) lists and attribute sets
+    can
+    [now](https://github.com/NixOS/nix/commit/6de33a9c675b187437a2e1abbcb290981a89ecb1)
+    be passed to builders in a non-lossy way. If the special attribute
+    `__structuredAttrs` is set to `true`, the other derivation
+    attributes are serialised in JSON format and made available to the
+    builder via the file .attrs.json in the builder’s temporary
+    directory. This obviates the need for `passAsFile` since JSON files
+    have no size restrictions, unlike process environments.
+    
+    [As a convenience to Bash
+    builders](https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699),
+    Nix writes a script named .attrs.sh to the builder’s directory that
+    initialises shell variables corresponding to all attributes that are
+    representable in Bash. This includes non-nested (associative)
+    arrays. For example, the attribute `hardening.format =
+                    true` ends up as the Bash associative array element
+    `${hardening[format]}`.
+
+  - Builders can
+    [now](https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3)
+    communicate what build phase they are in by writing messages to the
+    file descriptor specified in NIX\_LOG\_FD. The current phase is
+    shown by the `nix` progress indicator.
+
+  - In Linux sandbox builds, we
+    [now](https://github.com/NixOS/nix/commit/a2d92bb20e82a0957067ede60e91fab256948b41)
+    provide a default `/bin/sh` (namely `ash` from BusyBox).
+
+  - In structured attribute mode, `exportReferencesGraph`
+    [exports](https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a)
+    extended information about closures in JSON format. In particular,
+    it includes the sizes and hashes of paths. This is primarily useful
+    for NixOS image builders.
+
+  - Builds are
+    [now](https://github.com/NixOS/nix/commit/21948deed99a3295e4d5666e027a6ca42dc00b40)
+    killed as soon as Nix receives EOF on the builder’s stdout or
+    stderr. This fixes a bug that allowed builds to hang Nix
+    indefinitely, regardless of timeouts.
+
+  - The `sandbox-paths` configuration option can now specify optional
+    paths by appending a `?`, e.g. `/dev/nvidiactl?` will bind-mount
+    `/dev/nvidiactl` only if it exists.
+
+  - On Linux, builds are now executed in a user namespace with UID 1000
+    and GID 100.
+
+A number of significant internal changes were made:
+
+  - Nix no longer depends on Perl and all Perl components have been
+    rewritten in C++ or removed. The Perl bindings that used to be part
+    of Nix have been moved to a separate package, `nix-perl`.
+
+  - All `Store` classes are now thread-safe. `RemoteStore` supports
+    multiple concurrent connections to the daemon. This is primarily
+    useful in multi-threaded programs such as `hydra-queue-runner`.
+
+This release has contributions from Adrien Devresse, Alexander Ried,
+Alex Cruice, Alexey Shmalko, AmineChikhaoui, Andy Wingo, Aneesh Agrawal,
+Anthony Cowley, Armijn Hemel, aszlig, Ben Gamari, Benjamin Hipple,
+Benjamin Staffin, Benno Fünfstück, Bjørn Forsman, Brian McKenna, Charles
+Strahan, Chase Adams, Chris Martin, Christian Theune, Chris Warburton,
+Daiderd Jordan, Dan Connolly, Daniel Peebles, Dan Peebles, davidak,
+David McFarland, Dmitry Kalinkin, Domen Kožar, Eelco Dolstra, Emery
+Hemingway, Eric Litak, Eric Wolf, Fabian Schmitthenner, Frederik
+Rietdijk, Gabriel Gonzalez, Giorgio Gallo, Graham Christensen, Guillaume
+Maudoux, Harmen, Iavael, James Broadhead, James Earl Douglas, Janus
+Troelsen, Jeremy Shaw, Joachim Schiele, Joe Hermaszewski, Joel Moberg,
+Johannes 'fish' Ziemke, Jörg Thalheim, Jude Taylor, kballou, Keshav
+Kini, Kjetil Orbekk, Langston Barrett, Linus Heckemann, Ludovic Courtès,
+Manav Rathi, Marc Scholten, Markus Hauck, Matt Audesse, Matthew Bauer,
+Matthias Beyer, Matthieu Coudron, N1X, Nathan Zadoks, Neil Mayhew,
+Nicolas B. Pierron, Niklas Hambüchen, Nikolay Amiantov, Ole Jørgen
+Brønner, Orivej Desh, Peter Simons, Peter Stuart, Pyry Jahkola, regnat,
+Renzo Carbonara, Rhys, Robert Vollmert, Scott Olson, Scott R. Parish,
+Sergei Trofimovich, Shea Levy, Sheena Artrip, Spencer Baugh, Stefan
+Junker, Susan Potter, Thomas Tuegel, Timothy Allen, Tristan Hume, Tuomas
+Tynkkynen, tv, Tyson Whitehead, Vladimír Čunát, Will Dietz, wmertens,
+Wout Mertens, zimbatm and Zoran Plesivčak.
diff --git a/doc/manual/src/release-notes/rl-2.1.md b/doc/manual/src/release-notes/rl-2.1.md
new file mode 100644
index 000000000..08986ef9d
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.1.md
@@ -0,0 +1,49 @@
+# Release 2.1 (2018-09-02)
+
+This is primarily a bug fix release. It also reduces memory consumption
+in certain situations. In addition, it has the following new features:
+
+  - The Nix installer will no longer default to the Multi-User
+    installation for macOS. You can still [instruct the installer to run
+    in multi-user mode](#sect-multi-user-installation).
+
+  - The Nix installer now supports performing a Multi-User installation
+    for Linux computers which are running systemd. You can [select a
+    Multi-User installation](#sect-multi-user-installation) by passing
+    the `--daemon` flag to the installer: `sh <(curl
+                    https://nixos.org/nix/install) --daemon`.
+    
+    The multi-user installer cannot handle systems with SELinux. If your
+    system has SELinux enabled, you can [force the installer to run in
+    single-user mode](#sect-single-user-installation).
+
+  - New builtin functions: `builtins.bitAnd`, `builtins.bitOr`,
+    `builtins.bitXor`, `builtins.fromTOML`, `builtins.concatMap`,
+    `builtins.mapAttrs`.
+
+  - The S3 binary cache store now supports uploading NARs larger than 5
+    GiB.
+
+  - The S3 binary cache store now supports uploading to S3-compatible
+    services with the `endpoint` option.
+
+  - The flag `--fallback` is no longer required to recover from
+    disappeared NARs in binary caches.
+
+  - `nix-daemon` now respects `--store`.
+
+  - `nix run` now respects `nix-support/propagated-user-env-packages`.
+
+This release has contributions from Adrien Devresse, Aleksandr Pashkov,
+Alexandre Esteves, Amine Chikhaoui, Andrew Dunham, Asad Saeeduddin,
+aszlig, Ben Challenor, Ben Gamari, Benjamin Hipple, Bogdan Seniuc, Corey
+O'Connor, Daiderd Jordan, Daniel Peebles, Daniel Poelzleithner, Danylo
+Hlynskyi, Dmitry Kalinkin, Domen Kožar, Doug Beardsley, Eelco Dolstra,
+Erik Arvstedt, Félix Baylac-Jacqué, Gleb Peregud, Graham Christensen,
+Guillaume Maudoux, Ivan Kozik, John Arnold, Justin Humm, Linus
+Heckemann, Lorenzo Manacorda, Matthew Justin Bauer, Matthew O'Gorman,
+Maximilian Bosch, Michael Bishop, Michael Fiano, Michael Mercier,
+Michael Raskin, Michael Weiss, Nicolas Dudebout, Peter Simons, Ryan
+Trinkle, Samuel Dionne-Riel, Sean Seefried, Shea Levy, Symphorien Gibol,
+Tim Engler, Tim Sears, Tuomas Tynkkynen, volth, Will Dietz, Yorick van
+Pelt and zimbatm.
diff --git a/doc/manual/src/release-notes/rl-2.2.md b/doc/manual/src/release-notes/rl-2.2.md
new file mode 100644
index 000000000..3667a48dc
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.2.md
@@ -0,0 +1,82 @@
+# Release 2.2 (2019-01-11)
+
+This is primarily a bug fix release. It also has the following changes:
+
+  - In derivations that use structured attributes (i.e. that specify set
+    the `__structuredAttrs` attribute to `true` to cause all attributes
+    to be passed to the builder in JSON format), you can now specify
+    closure checks per output, e.g.:
+    
+        outputChecks."out" = {
+          # The closure of 'out' must not be larger than 256 MiB.
+          maxClosureSize = 256 * 1024 * 1024;
+        
+          # It must not refer to C compiler or to the 'dev' output.
+          disallowedRequisites = [ stdenv.cc "dev" ];
+        };
+        
+        outputChecks."dev" = {
+          # The 'dev' output must not be larger than 128 KiB.
+          maxSize = 128 * 1024;
+        };
+
+  - The derivation attribute `requiredSystemFeatures` is now enforced
+    for local builds, and not just to route builds to remote builders.
+    The supported features of a machine can be specified through the
+    configuration setting `system-features`.
+    
+    By default, `system-features` includes `kvm` if `/dev/kvm` exists.
+    For compatibility, it also includes the pseudo-features
+    `nixos-test`, `benchmark` and `big-parallel` which are used by
+    Nixpkgs to route builds to particular Hydra build machines.
+
+  - Sandbox builds are now enabled by default on Linux.
+
+  - The new command `nix doctor` shows potential issues with your Nix
+    installation.
+
+  - The `fetchGit` builtin function now uses a caching scheme that puts
+    different remote repositories in distinct local repositories, rather
+    than a single shared repository. This may require more disk space
+    but is faster.
+
+  - The `dirOf` builtin function now works on relative paths.
+
+  - Nix now supports [SRI hashes](https://www.w3.org/TR/SRI/), allowing
+    the hash algorithm and hash to be specified in a single string. For
+    example, you can write:
+    
+        import <nix/fetchurl.nix> {
+          url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
+          hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ=";
+        };
+    
+    instead of
+    
+        import <nix/fetchurl.nix> {
+          url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
+          sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4";
+        };
+    
+    In fixed-output derivations, the `outputHashAlgo` attribute is no
+    longer mandatory if `outputHash` specifies the hash.
+    
+    `nix hash-file` and `nix
+                    hash-path` now print hashes in SRI format by default. They also use
+    SHA-256 by default instead of SHA-512 because that's what we use
+    most of the time in Nixpkgs.
+
+  - Integers are now 64 bits on all platforms.
+
+  - The evaluator now prints profiling statistics (enabled via the
+    NIX\_SHOW\_STATS and NIX\_COUNT\_CALLS environment variables) in
+    JSON format.
+
+  - The option `--xml` in `nix-store
+                    --query` has been removed. Instead, there now is an option
+    `--graphml` to output the dependency graph in GraphML format.
+
+  - All `nix-*` commands are now symlinks to `nix`. This saves a bit of
+    disk space.
+
+  - `nix repl` now uses `libeditline` or `libreadline`.
diff --git a/doc/manual/src/release-notes/rl-2.3.md b/doc/manual/src/release-notes/rl-2.3.md
new file mode 100644
index 000000000..a4bb6ebb0
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.3.md
@@ -0,0 +1,44 @@
+# Release 2.3 (2019-09-04)
+
+This is primarily a bug fix release. However, it makes some incompatible
+changes:
+
+  - Nix now uses BSD file locks instead of POSIX file locks. Because of
+    this, you should not use Nix 2.3 and previous releases at the same
+    time on a Nix store.
+
+It also has the following changes:
+
+  - `builtins.fetchGit`'s `ref` argument now allows specifying an
+    absolute remote ref. Nix will automatically prefix `ref` with
+    `refs/heads` only if `ref` doesn't already begin with `refs/`.
+
+  - The installer now enables sandboxing by default on Linux when the
+    system has the necessary kernel support.
+
+  - The `max-jobs` setting now defaults to 1.
+
+  - New builtin functions: `builtins.isPath`, `builtins.hashFile`.
+
+  - The `nix` command has a new `--print-build-logs` (`-L`) flag to
+    print build log output to stderr, rather than showing the last log
+    line in the progress bar. To distinguish between concurrent builds,
+    log lines are prefixed by the name of the package.
+
+  - Builds are now executed in a pseudo-terminal, and the TERM
+    environment variable is set to `xterm-256color`. This allows many
+    programs (e.g. `gcc`, `clang`, `cmake`) to print colorized log
+    output.
+
+  - Add `--no-net` convenience flag. This flag disables substituters;
+    sets the `tarball-ttl` setting to infinity (ensuring that any
+    previously downloaded files are considered current); and disables
+    retrying downloads and sets the connection timeout to the minimum.
+    This flag is enabled automatically if there are no configured
+    non-loopback network interfaces.
+
+  - Add a `post-build-hook` setting to run a program after a build has
+    succeeded.
+
+  - Add a `trace-function-calls` setting to log the duration of Nix
+    function calls to stderr.

From 942cd687f9ae190fc9a989a46c3207bd38377ade Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jul 2020 23:44:34 +0200
Subject: [PATCH 024/882] Remove libxml2 / libxslt prerequisites

---
 doc/manual/installation/prerequisites-source.xml    | 12 ------------
 doc/manual/src/installation/prerequisites-source.md |  9 ---------
 2 files changed, 21 deletions(-)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index fa6da9b1e..77955eecc 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -74,18 +74,6 @@
   1.14.0 or higher. It can be obtained from the its repository
   <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem>
 
-  <listitem><para>The <command>xmllint</command> and
-  <command>xsltproc</command> programs to build this manual and the
-  man-pages.  These are part of the <literal>libxml2</literal> and
-  <literal>libxslt</literal> packages, respectively.  You also need
-  the <link
-  xlink:href="http://docbook.sourceforge.net/projects/xsl/">DocBook
-  XSL stylesheets</link> and optionally the <link
-  xlink:href="http://www.docbook.org/schemas/5x"> DocBook 5.0 RELAX NG
-  schemas</link>.  Note that these are only required if you modify the
-  manual sources or when you are building from the Git
-  repository.</para></listitem>
-
   <listitem><para>Recent versions of Bison and Flex to build the
   parser.  (This is because Nix needs GLR support in Bison and
   reentrancy support in Flex.)  For Bison, you need version 2.6, which
diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index 311961fe9..69b7c5a5e 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -54,15 +54,6 @@
     obtained from the its repository
     <https://github.com/troglobit/editline>.
 
-  - The `xmllint` and `xsltproc` programs to build this manual and the
-    man-pages. These are part of the `libxml2` and `libxslt` packages,
-    respectively. You also need the [DocBook XSL
-    stylesheets](http://docbook.sourceforge.net/projects/xsl/) and
-    optionally the [DocBook 5.0 RELAX NG
-    schemas](http://www.docbook.org/schemas/5x). Note that these are
-    only required if you modify the manual sources or when you are
-    building from the Git repository.
-
   - Recent versions of Bison and Flex to build the parser. (This is
     because Nix needs GLR support in Bison and reentrancy support in
     Flex.) For Bison, you need version 2.6, which can be obtained from

From c20c0823838d257b1e18e71c307f53afac0d2b39 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 10:38:19 +0200
Subject: [PATCH 025/882] <envar> -> <literal>

---
 doc/manual/advanced-topics/cores-vs-jobs.xml  | 10 +++---
 .../advanced-topics/distributed-builds.xml    |  2 +-
 doc/manual/command-ref/conf-file.xml          | 20 +++++------
 doc/manual/command-ref/env-common.xml         | 34 +++++++++----------
 doc/manual/command-ref/nix-copy-closure.xml   |  2 +-
 doc/manual/command-ref/nix-env.xml            |  4 +--
 doc/manual/command-ref/nix-instantiate.xml    |  4 +--
 doc/manual/command-ref/nix-shell.xml          | 10 +++---
 doc/manual/command-ref/nix-store.xml          |  2 +-
 doc/manual/command-ref/opt-common.xml         |  6 ++--
 .../expressions/advanced-attributes.xml       |  6 ++--
 doc/manual/expressions/build-script.xml       | 14 ++++----
 doc/manual/expressions/derivations.xml        | 16 ++++-----
 doc/manual/expressions/generic-builder.xml    |  8 ++---
 doc/manual/expressions/language-values.xml    |  2 +-
 .../expressions/simple-building-testing.xml   |  2 +-
 doc/manual/installation/env-variables.xml     | 10 +++---
 doc/manual/installation/installing-binary.xml |  2 +-
 doc/manual/installation/multi-user.xml        |  2 +-
 doc/manual/introduction/about-nix.xml         |  2 +-
 doc/manual/packages/basic-package-mgmt.xml    |  2 +-
 doc/manual/packages/profiles.xml              |  8 ++---
 doc/manual/release-notes/rl-0.10.xml          |  6 ++--
 doc/manual/release-notes/rl-0.12.xml          |  4 +--
 doc/manual/release-notes/rl-0.16.xml          |  2 +-
 doc/manual/release-notes/rl-1.0.xml           |  2 +-
 doc/manual/release-notes/rl-1.2.xml           |  2 +-
 doc/manual/release-notes/rl-1.6.xml           |  2 +-
 doc/manual/release-notes/rl-1.8.xml           |  2 +-
 doc/manual/release-notes/rl-1.9.xml           |  6 ++--
 doc/manual/release-notes/rl-2.0.xml           | 22 ++++++------
 doc/manual/release-notes/rl-2.2.xml           |  4 +--
 doc/manual/release-notes/rl-2.3.xml           |  2 +-
 33 files changed, 111 insertions(+), 111 deletions(-)

diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml
index 4d58ac7c5..a75dea37f 100644
--- a/doc/manual/advanced-topics/cores-vs-jobs.xml
+++ b/doc/manual/advanced-topics/cores-vs-jobs.xml
@@ -31,13 +31,13 @@ they are, how they interact, and their configuration trade-offs.</para>
 </variablelist>
 
 <para>The <xref linkend="conf-cores" /> setting determines the value of
-<envar>NIX_BUILD_CORES</envar>. <envar>NIX_BUILD_CORES</envar> is equal
+<literal>NIX_BUILD_CORES</literal>. <literal>NIX_BUILD_CORES</literal> is equal
 to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
-equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar>
+equals <literal>0</literal>, in which case <literal>NIX_BUILD_CORES</literal>
 will be the total number of cores in the system.</para>
 
 <para>The maximum number of consumed cores is a simple multiplication,
-<xref linkend="conf-max-jobs" /> * <envar>NIX_BUILD_CORES</envar>.</para>
+<xref linkend="conf-max-jobs" /> * <literal>NIX_BUILD_CORES</literal>.</para>
 
 <para>The balance on how to set these two independent variables depends
 upon each builder's workload and hardware. Here are a few example
@@ -49,7 +49,7 @@ scenarios on a machine with 24 cores:</para>
     <tr>
       <th><xref linkend="conf-max-jobs" /></th>
       <th><xref linkend="conf-cores" /></th>
-      <th><envar>NIX_BUILD_CORES</envar></th>
+      <th><literal>NIX_BUILD_CORES</literal></th>
       <th>Maximum Processes</th>
       <th>Result</th>
     </tr>
@@ -116,6 +116,6 @@ scenarios on a machine with 24 cores:</para>
 
 <para>It is up to the derivations' build script to respect
 host's requested cores-per-build by following the value of the
-<envar>NIX_BUILD_CORES</envar> environment variable.</para>
+<literal>NIX_BUILD_CORES</literal> environment variable.</para>
 
 </chapter>
diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index 9ac4a92cd..a649b6f8d 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -43,7 +43,7 @@ bash: nix-store: command not found
 error: cannot connect to 'mac'
 </screen>
 
-then you need to ensure that the <envar>PATH</envar> of
+then you need to ensure that the <literal>PATH</literal> of
 non-interactive login shells contains Nix.</para>
 
 <warning><para>If you are building via the Nix daemon, it is the Nix
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1fa74a143..ff9c7d46c 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -25,20 +25,20 @@
 <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
 (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
 <filename>$NIX_CONF_DIR/nix.conf</filename> if
-<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
+<literal>NIX_CONF_DIR</literal> is set. Values loaded in this file are not forwarded to the Nix daemon. The
 client assumes that the daemon has already loaded them.
 </para>
 
 <para>User-specific configuration files:</para>
 
 <para>
-  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
+  If <literal>NIX_USER_CONF_FILES</literal> is set, then each path separated by
   <literal>:</literal> will be loaded in reverse order.
 </para>
 
 <para>
   Otherwise it will look for <filename>nix/nix.conf</filename> files in
-  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
+  <literal>XDG_CONFIG_DIRS</literal> and <literal>XDG_CONFIG_HOME</literal>.
 
   The default location is <filename>$HOME/.config/nix.conf</filename> if
   those environment variables are unset.
@@ -195,8 +195,8 @@ false</literal>.</para>
 
     <para>If the build users group is empty, builds will be performed
     under the uid of the Nix process (that is, the uid of the caller
-    if <envar>NIX_REMOTE</envar> is empty, the uid under which the Nix
-    daemon runs if <envar>NIX_REMOTE</envar> is
+    if <literal>NIX_REMOTE</literal> is empty, the uid under which the Nix
+    daemon runs if <literal>NIX_REMOTE</literal> is
     <literal>daemon</literal>).  Obviously, this should not be used in
     multi-user settings with untrusted users.</para>
 
@@ -231,7 +231,7 @@ false</literal>.</para>
   <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term>
 
     <listitem><para>Sets the value of the
-    <envar>NIX_BUILD_CORES</envar> environment variable in the
+    <literal>NIX_BUILD_CORES</literal> environment variable in the
     invocation of builders.  Builders can use this variable at their
     discretion to control the maximum amount of parallelism.  For
     instance, in Nixpkgs, if the derivation attribute
@@ -694,7 +694,7 @@ password <replaceable>my-password</replaceable>
 
       <variablelist>
         <varlistentry>
-          <term><envar>DRV_PATH</envar></term>
+          <term><literal>DRV_PATH</literal></term>
           <listitem>
             <para>The derivation for the built paths.</para>
             <para>Example:
@@ -704,7 +704,7 @@ password <replaceable>my-password</replaceable>
         </varlistentry>
 
         <varlistentry>
-          <term><envar>OUT_PATHS</envar></term>
+          <term><literal>OUT_PATHS</literal></term>
           <listitem>
             <para>Output paths of the built derivation, separated by a space character.</para>
             <para>Example:
@@ -754,7 +754,7 @@ password <replaceable>my-password</replaceable>
 
       <para>If set to <literal>true</literal>, the Nix evaluator will
       not allow access to any files outside of the Nix search path (as
-      set via the <envar>NIX_PATH</envar> environment variable or the
+      set via the <literal>NIX_PATH</literal> environment variable or the
       <option>-I</option> option), or to URIs outside of
       <option>allowed-uri</option>. The default is
       <literal>false</literal>.</para>
@@ -958,7 +958,7 @@ requiredSystemFeatures = [ "kvm" ];
       <para>Nix caches tarballs in
       <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para>
 
-      <para>Files fetched via <envar>NIX_PATH</envar>,
+      <para>Files fetched via <literal>NIX_PATH</literal>,
       <function>fetchGit</function>, <function>fetchMercurial</function>,
       <function>fetchTarball</function>, and <function>fetchurl</function>
       respect this TTL.
diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index 8466cc463..3196cbbd2 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -11,7 +11,7 @@
 
 <variablelist xml:id="env-common">
 
-<varlistentry><term><envar>IN_NIX_SHELL</envar></term>
+<varlistentry><term><literal>IN_NIX_SHELL</literal></term>
 
   <listitem><para>Indicator that tells if the current environment was set up by
   <command>nix-shell</command>.  Since Nix 2.0 the values are
@@ -19,7 +19,7 @@
 
 </varlistentry>
 
-<varlistentry xml:id="env-NIX_PATH"><term><envar>NIX_PATH</envar></term>
+<varlistentry xml:id="env-NIX_PATH"><term><literal>NIX_PATH</literal></term>
 
   <listitem>
 
@@ -50,7 +50,7 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     interpreted as the URL of a tarball that will be downloaded and
     unpacked to a temporary location. The tarball must consist of a
     single top-level directory. For example, setting
-    <envar>NIX_PATH</envar> to
+    <literal>NIX_PATH</literal> to
 
     <screen>
 nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
@@ -65,12 +65,12 @@ nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
 
     <para>The search path can be extended using the <option
     linkend="opt-I">-I</option> option, which takes precedence over
-    <envar>NIX_PATH</envar>.</para></listitem>
+    <literal>NIX_PATH</literal>.</para></listitem>
 
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
+<varlistentry><term><literal>NIX_IGNORE_SYMLINK_STORE</literal></term>
 
   <listitem>
 
@@ -84,7 +84,7 @@ nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
   except when builds are deployed to machines where
   <filename>/nix/store</filename> resolves differently.  If you are
   sure that you’re not going to do that, you can set
-  <envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para>
+  <literal>NIX_IGNORE_SYMLINK_STORE</literal> to <literal>1</literal>.</para>
 
   <para>Note that if you’re symlinking the Nix store so that you can
   put it on another file system than the root file system, on Linux
@@ -102,7 +102,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_STORE_DIR</envar></term>
+<varlistentry><term><literal>NIX_STORE_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix store (default
   <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
@@ -110,7 +110,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_DATA_DIR</envar></term>
+<varlistentry><term><literal>NIX_DATA_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix static data
   directory (default
@@ -119,7 +119,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_LOG_DIR</envar></term>
+<varlistentry><term><literal>NIX_LOG_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix log directory
   (default <filename><replaceable>prefix</replaceable>/var/log/nix</filename>).</para></listitem>
@@ -127,7 +127,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_STATE_DIR</envar></term>
+<varlistentry><term><literal>NIX_STATE_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix state directory
   (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
@@ -135,7 +135,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_CONF_DIR</envar></term>
+<varlistentry><term><literal>NIX_CONF_DIR</literal></term>
 
   <listitem><para>Overrides the location of the system Nix configuration
   directory (default
@@ -143,7 +143,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 </varlistentry>
 
-<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
+<varlistentry><term><literal>NIX_USER_CONF_FILES</literal></term>
 
   <listitem><para>Overrides the location of the user Nix configuration files
   to load from (defaults to the XDG spec locations). The variable is treated
@@ -151,7 +151,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 </varlistentry>
 
-<varlistentry><term><envar>TMPDIR</envar></term>
+<varlistentry><term><literal>TMPDIR</literal></term>
 
   <listitem><para>Use the specified directory to store temporary
   files.  In particular, this includes temporary build directories;
@@ -161,7 +161,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry xml:id="envar-remote"><term><envar>NIX_REMOTE</envar></term>
+<varlistentry xml:id="envar-remote"><term><literal>NIX_REMOTE</literal></term>
 
   <listitem><para>This variable should be set to
   <literal>daemon</literal> if you want to use the Nix daemon to
@@ -174,7 +174,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_SHOW_STATS</envar></term>
+<varlistentry><term><literal>NIX_SHOW_STATS</literal></term>
 
   <listitem><para>If set to <literal>1</literal>, Nix will print some
   evaluation statistics, such as the number of values
@@ -183,7 +183,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>NIX_COUNT_CALLS</envar></term>
+<varlistentry><term><literal>NIX_COUNT_CALLS</literal></term>
 
   <listitem><para>If set to <literal>1</literal>, Nix will print how
   often functions were called during Nix expression evaluation.  This
@@ -192,7 +192,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry><term><envar>GC_INITIAL_HEAP_SIZE</envar></term>
+<varlistentry><term><literal>GC_INITIAL_HEAP_SIZE</literal></term>
 
   <listitem><para>If Nix has been configured to use the Boehm garbage
   collector, this variable sets the initial size of the heap in bytes.
diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml
index e6dcf180a..8c07984fb 100644
--- a/doc/manual/command-ref/nix-copy-closure.xml
+++ b/doc/manual/command-ref/nix-copy-closure.xml
@@ -129,7 +129,7 @@ those paths.  If this bothers you, use
 
 <variablelist>
 
-  <varlistentry><term><envar>NIX_SSHOPTS</envar></term>
+  <varlistentry><term><literal>NIX_SSHOPTS</literal></term>
 
     <listitem><para>Additional options to be passed to
     <command>ssh</command> on the command line.</para></listitem>
diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 55f25d959..4dedd90ca 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -277,7 +277,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
     <listitem><para>A symbolic link to the user's current profile.  By
     default, this symlink points to
     <filename><replaceable>prefix</replaceable>/var/nix/profiles/default</filename>.
-    The <envar>PATH</envar> environment variable should include
+    The <literal>PATH</literal> environment variable should include
     <filename>~/.nix-profile/bin</filename> for the user environment
     to be visible to the user.</para></listitem>
 
@@ -1485,7 +1485,7 @@ error: no generation older than the current (91) exists</screen>
 
 <variablelist>
 
-  <varlistentry><term><envar>NIX_PROFILE</envar></term>
+  <varlistentry><term><literal>NIX_PROFILE</literal></term>
 
     <listitem><para>Location of the Nix profile.  Defaults to the
     target of the symlink <filename>~/.nix-profile</filename>, if it
diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index 53f06aed1..4ff967ecf 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -106,10 +106,10 @@ input.</para>
   <varlistentry><term><option>--find-file</option></term>
 
     <listitem><para>Look up the given files in Nix’s search path (as
-    specified by the <envar linkend="env-NIX_PATH">NIX_PATH</envar>
+    specified by the <envar linkend="env-NIX_PATH">NIX_PATH</literal>
     environment variable).  If found, print the corresponding absolute
     paths on standard output.  For instance, if
-    <envar>NIX_PATH</envar> is
+    <literal>NIX_PATH</literal> is
     <literal>nixpkgs=/home/alice/nixpkgs</literal>, then
     <literal>nix-instantiate --find-file nixpkgs/default.nix</literal>
     will print
diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 2fef323c5..de9755d58 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -141,8 +141,8 @@ also <xref linkend="sec-common-options" />.</phrase></para>
     almost entirely cleared before the interactive shell is started,
     so you get an environment that more closely corresponds to the
     “real” Nix build.  A few variables, in particular
-    <envar>HOME</envar>, <envar>USER</envar> and
-    <envar>DISPLAY</envar>, are retained.  Note that
+    <literal>HOME</literal>, <literal>USER</literal> and
+    <literal>DISPLAY</literal>, are retained.  Note that
     <filename>~/.bashrc</filename> and (depending on your Bash
     installation) <filename>/etc/bashrc</filename> are still sourced,
     so any variables set there will affect the interactive
@@ -193,10 +193,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <variablelist>
 
-  <varlistentry><term><envar>NIX_BUILD_SHELL</envar></term>
+  <varlistentry><term><literal>NIX_BUILD_SHELL</literal></term>
 
     <listitem><para>Shell used to start the interactive environment.
-    Defaults to the <command>bash</command> found in <envar>PATH</envar>.</para></listitem>
+    Defaults to the <command>bash</command> found in <literal>PATH</literal>.</para></listitem>
 
   </varlistentry>
 
@@ -254,7 +254,7 @@ $ nix-shell -p sqlite 'git.override { withManual = false; }'
 
 The <command>-p</command> flag looks up Nixpkgs in the Nix search
 path. You can override it by passing <option>-I</option> or setting
-<envar>NIX_PATH</envar>. For example, the following gives you a shell
+<literal>NIX_PATH</literal>. For example, the following gives you a shell
 containing the Pan package from a specific revision of Nixpkgs:
 
 <screen>
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index d71f9d8e4..459100984 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -1433,7 +1433,7 @@ loads it into the Nix database.</para>
 <para>The operation <option>--print-env</option> prints out the
 environment of a derivation in a format that can be evaluated by a
 shell.  The command line arguments of the builder are placed in the
-variable <envar>_args</envar>.</para>
+variable <literal>_args</literal>.</para>
 
 </refsection>
 
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index a68eef1d0..150e732ff 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -160,7 +160,7 @@
 
 <varlistentry xml:id="opt-cores"><term><option>--cores</option></term>
 
-  <listitem><para>Sets the value of the <envar>NIX_BUILD_CORES</envar>
+  <listitem><para>Sets the value of the <literal>NIX_BUILD_CORES</literal>
   environment variable in the invocation of builders.  Builders can
   use this variable at their discretion to control the maximum amount
   of parallelism.  For instance, in Nixpkgs, if the derivation
@@ -370,10 +370,10 @@
 
   <listitem><para>Add a path to the Nix expression search path.  This
   option may be given multiple times.  See the <envar
-  linkend="env-NIX_PATH">NIX_PATH</envar> environment variable for
+  linkend="env-NIX_PATH">NIX_PATH</literal> environment variable for
   information on the semantics of the Nix search path.  Paths added
   through <option>-I</option> take precedence over
-  <envar>NIX_PATH</envar>.</para></listitem>
+  <literal>NIX_PATH</literal>.</para></listitem>
 
 </varlistentry>
 
diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 5759ff50e..3a0413ceb 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -139,7 +139,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 </programlisting>
 
     to make it use the proxy server configuration specified by the
-    user in the environment variables <envar>http_proxy</envar> and
+    user in the environment variables <literal>http_proxy</literal> and
     friends.</para>
 
     <para>This attribute is only allowed in <link
@@ -294,12 +294,12 @@ big = "a very long string";
     </programlisting>
 
     then when the builder runs, the environment variable
-    <envar>bigPath</envar> will contain the absolute path to a
+    <literal>bigPath</literal> will contain the absolute path to a
     temporary file containing <literal>a very long
     string</literal>. That is, for any attribute
     <replaceable>x</replaceable> listed in
     <varname>passAsFile</varname>, Nix will pass an environment
-    variable <envar><replaceable>x</replaceable>Path</envar> holding
+    variable <literal><replaceable>x</replaceable>Path</literal> holding
     the path of the file containing the value of attribute
     <replaceable>x</replaceable>. This is useful when you need to pass
     large strings to a builder, since most operating systems impose a
diff --git a/doc/manual/expressions/build-script.xml b/doc/manual/expressions/build-script.xml
index 7bad8f808..44264239d 100644
--- a/doc/manual/expressions/build-script.xml
+++ b/doc/manual/expressions/build-script.xml
@@ -35,19 +35,19 @@ steps:</para>
 
     <para>When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the
-    derivation).  For instance, the <envar>PATH</envar> variable is
+    derivation).  For instance, the <literal>PATH</literal> variable is
     empty<footnote><para>Actually, it's initialised to
     <filename>/path-not-set</filename> to prevent Bash from setting it
     to a default value.</para></footnote>.  This is done to prevent
     undeclared inputs from being used in the build process.  If for
-    example the <envar>PATH</envar> contained
+    example the <literal>PATH</literal> contained
     <filename>/usr/bin</filename>, then you might accidentally use
     <filename>/usr/bin/gcc</filename>.</para>
 
     <para>So the first step is to set up the environment.  This is
     done by calling the <filename>setup</filename> script of the
     standard environment.  The environment variable
-    <envar>stdenv</envar> points to the location of the standard
+    <literal>stdenv</literal> points to the location of the standard
     environment being used.  (It wasn't specified explicitly as an
     attribute in <xref linkend='ex-hello-nix' />, but
     <varname>mkDerivation</varname> adds it automatically.)</para>
@@ -57,7 +57,7 @@ steps:</para>
   <callout arearefs='ex-hello-builder-co-2'>
 
     <para>Since Hello needs Perl, we have to make sure that Perl is in
-    the <envar>PATH</envar>.  The <envar>perl</envar> environment
+    the <literal>PATH</literal>.  The <literal>perl</literal> environment
     variable points to the location of the Perl package (since it
     was passed in as an attribute to the derivation), so
     <filename><replaceable>$perl</replaceable>/bin</filename> is the
@@ -70,7 +70,7 @@ steps:</para>
     <para>Now we have to unpack the sources.  The
     <varname>src</varname> attribute was bound to the result of
     fetching the Hello source tarball from the network, so the
-    <envar>src</envar> environment variable points to the location in
+    <literal>src</literal> environment variable points to the location in
     the Nix store to which the tarball was downloaded.  After
     unpacking, we <command>cd</command> to the resulting source
     directory.</para>
@@ -93,7 +93,7 @@ steps:</para>
     <filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>.
     Nix computes this path by cryptographically hashing all attributes
     of the derivation.  The path is passed to the builder through the
-    <envar>out</envar> environment variable.  So here we give
+    <literal>out</literal> environment variable.  So here we give
     <filename>configure</filename> the parameter
     <literal>--prefix=$out</literal> to cause Hello to be installed in
     the expected location.</para>
@@ -103,7 +103,7 @@ steps:</para>
   <callout arearefs='ex-hello-builder-co-5'>
 
     <para>Finally we build Hello (<literal>make</literal>) and install
-    it into the location specified by <envar>out</envar>
+    it into the location specified by <literal>out</literal>
     (<literal>make install</literal>).</para>
 
   </callout>
diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml
index 6f6297565..a11de0088 100644
--- a/doc/manual/expressions/derivations.xml
+++ b/doc/manual/expressions/derivations.xml
@@ -122,7 +122,7 @@ the Nixpkgs manual for details.</para>
 <itemizedlist>
 
   <listitem><para>A temporary directory is created under the directory
-  specified by <envar>TMPDIR</envar> (default
+  specified by <literal>TMPDIR</literal> (default
   <filename>/tmp</filename>) where the build will take place.  The
   current directory is changed to this directory.</para></listitem>
 
@@ -133,29 +133,29 @@ the Nixpkgs manual for details.</para>
 
   <itemizedlist>
 
-    <listitem><para><envar>NIX_BUILD_TOP</envar> contains the path of
+    <listitem><para><literal>NIX_BUILD_TOP</literal> contains the path of
     the temporary directory for this build.</para></listitem>
 
-    <listitem><para>Also, <envar>TMPDIR</envar>,
-    <envar>TEMPDIR</envar>, <envar>TMP</envar>, <envar>TEMP</envar>
+    <listitem><para>Also, <literal>TMPDIR</literal>,
+    <literal>TEMPDIR</literal>, <literal>TMP</literal>, <literal>TEMP</literal>
     are set to point to the temporary directory.  This is to prevent
     the builder from accidentally writing temporary files anywhere
     else.  Doing so might cause interference by other
     processes.</para></listitem>
 
-    <listitem><para><envar>PATH</envar> is set to
+    <listitem><para><literal>PATH</literal> is set to
     <filename>/path-not-set</filename> to prevent shells from
     initialising it to their built-in default value.</para></listitem>
 
-    <listitem><para><envar>HOME</envar> is set to
+    <listitem><para><literal>HOME</literal> is set to
     <filename>/homeless-shelter</filename> to prevent programs from
     using <filename>/etc/passwd</filename> or the like to find the
     user's home directory, which could cause impurity.  Usually, when
-    <envar>HOME</envar> is set, it is used as the location of the home
+    <literal>HOME</literal> is set, it is used as the location of the home
     directory, even if it points to a non-existent
     path.</para></listitem>
 
-    <listitem><para><envar>NIX_STORE</envar> is set to the path of the
+    <listitem><para><literal>NIX_STORE</literal> is set to the path of the
     top-level Nix store directory (typically,
     <filename>/nix/store</filename>).</para></listitem>
 
diff --git a/doc/manual/expressions/generic-builder.xml b/doc/manual/expressions/generic-builder.xml
index db7ff405d..16b0268a7 100644
--- a/doc/manual/expressions/generic-builder.xml
+++ b/doc/manual/expressions/generic-builder.xml
@@ -38,18 +38,18 @@ genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting>
 
   <callout arearefs='ex-hello-builder2-co-1'>
 
-    <para>The <envar>buildInputs</envar> variable tells
+    <para>The <literal>buildInputs</literal> variable tells
     <filename>setup</filename> to use the indicated packages as
     <quote>inputs</quote>.  This means that if a package provides a
     <filename>bin</filename> subdirectory, it's added to
-    <envar>PATH</envar>; if it has a <filename>include</filename>
+    <literal>PATH</literal>; if it has a <filename>include</filename>
     subdirectory, it's added to GCC's header search path; and so
     on.<footnote><para>How does it work? <filename>setup</filename>
     tries to source the file
     <filename><replaceable>pkg</replaceable>/nix-support/setup-hook</filename>
     of all dependencies.  These “setup hooks” can then set up whatever
     environment variables they want; for instance, the setup hook for
-    Perl sets the <envar>PERL5LIB</envar> environment variable to
+    Perl sets the <literal>PERL5LIB</literal> environment variable to
     contain the <filename>lib/site_perl</filename> directories of all
     inputs.</para></footnote>
     </para>
@@ -78,7 +78,7 @@ genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting>
 </calloutlist>
 
 <para>Discerning readers will note that the
-<envar>buildInputs</envar> could just as well have been set in the Nix
+<literal>buildInputs</literal> could just as well have been set in the Nix
 expression, like this:
 
 <programlisting>
diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml
index bb2090c88..4a72c67a8 100644
--- a/doc/manual/expressions/language-values.xml
+++ b/doc/manual/expressions/language-values.xml
@@ -176,7 +176,7 @@ stdenv.mkDerivation {
   <para>Paths can also be specified between angle brackets, e.g.
   <literal>&lt;nixpkgs&gt;</literal>. This means that the directories
   listed in the environment variable
-  <envar linkend="env-NIX_PATH">NIX_PATH</envar> will be searched
+  <envar linkend="env-NIX_PATH">NIX_PATH</literal> will be searched
   for the given file or directory name.
   </para>
 
diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml
index ce0a1636d..33a802e83 100644
--- a/doc/manual/expressions/simple-building-testing.xml
+++ b/doc/manual/expressions/simple-building-testing.xml
@@ -45,7 +45,7 @@ name.</para>
 
 <para>Nix has transactional semantics.  Once a build finishes
 successfully, Nix makes a note of this in its database: it registers
-that the path denoted by <envar>out</envar> is now
+that the path denoted by <literal>out</literal> is now
 <quote>valid</quote>.  If you try to build the derivation again, Nix
 will see that the path is already valid and finish immediately.  If a
 build fails, either because it returns a non-zero exit code, because
diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
index cc52f5b4a..436f15f31 100644
--- a/doc/manual/installation/env-variables.xml
+++ b/doc/manual/installation/env-variables.xml
@@ -7,7 +7,7 @@
 <title>Environment Variables</title>
 
 <para>To use Nix, some environment variables should be set.  In
-particular, <envar>PATH</envar> should contain the directories
+particular, <literal>PATH</literal> should contain the directories
 <filename><replaceable>prefix</replaceable>/bin</filename> and
 <filename>~/.nix-profile/bin</filename>.  The first directory contains
 the Nix tools themselves, while <filename>~/.nix-profile</filename> is
@@ -23,15 +23,15 @@ source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>
 
 <section xml:id="sec-nix-ssl-cert-file">
 
-<title><envar>NIX_SSL_CERT_FILE</envar></title>
+<title><literal>NIX_SSL_CERT_FILE</literal></title>
 
 <para>If you need to specify a custom certificate bundle to account
 for an HTTPS-intercepting man in the middle proxy, you must specify
 the path to the certificate bundle in the environment variable
-<envar>NIX_SSL_CERT_FILE</envar>.</para>
+<literal>NIX_SSL_CERT_FILE</literal>.</para>
 
 
-<para>If you don't specify a <envar>NIX_SSL_CERT_FILE</envar>
+<para>If you don't specify a <literal>NIX_SSL_CERT_FILE</literal>
 manually, Nix will install and use its own certificate
 bundle.</para>
 
@@ -56,7 +56,7 @@ the Nix installer will detect the presense of Nix configuration, and
 abort.</para></note>
 
 <section xml:id="sec-nix-ssl-cert-file-with-nix-daemon-and-macos">
-<title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title>
+<title><literal>NIX_SSL_CERT_FILE</literal> with macOS and the Nix daemon</title>
 
 <para>On macOS you must specify the environment variable for the Nix
 daemon service, then restart it:</para>
diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 64c7a37fb..ad47ce8b9 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -61,7 +61,7 @@ The install script will modify the first writable file from amongst
 <filename>.bash_profile</filename>, <filename>.bash_login</filename>
 and <filename>.profile</filename> to source
 <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
+the <literal>NIX_INSTALLER_NO_MODIFY_PROFILE</literal> environment
 variable before executing the install script to disable this
 behaviour.
 </para>
diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml
index 69ae1ef27..331b5f128 100644
--- a/doc/manual/installation/multi-user.xml
+++ b/doc/manual/installation/multi-user.xml
@@ -69,7 +69,7 @@ You’ll want to put that line somewhere in your system’s boot
 scripts.</para>
 
 <para>To let unprivileged users use the daemon, they should set the
-<link linkend="envar-remote"><envar>NIX_REMOTE</envar> environment
+<link linkend="envar-remote"><literal>NIX_REMOTE</literal> environment
 variable</link> to <literal>daemon</literal>.  So you should put a
 line like
 
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index c21ed34dd..ec6f1ca3f 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -76,7 +76,7 @@ extremely well.</para>
 <para>Nix has multi-user support.  This means that non-privileged
 users can securely install software.  Each user can have a different
 <emphasis>profile</emphasis>, a set of packages in the Nix store that
-appear in the user’s <envar>PATH</envar>.  If a user installs a
+appear in the user’s <literal>PATH</literal>.  If a user installs a
 package that another user has already installed previously, the
 package won’t be built or downloaded a second time.  At the same time,
 it is not possible for one user to inject a Trojan horse into a
diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml
index 0f21297f3..758a4500d 100644
--- a/doc/manual/packages/basic-package-mgmt.xml
+++ b/doc/manual/packages/basic-package-mgmt.xml
@@ -16,7 +16,7 @@ on the set of installed applications.  That is, there might be lots of
 applications present on the system (possibly in many different
 versions), but users can have a specific selection of those active —
 where “active” just means that it appears in a directory
-in the user’s <envar>PATH</envar>.  Such a view on the set of
+in the user’s <literal>PATH</literal>.  Such a view on the set of
 installed applications is called a <emphasis>user
 environment</emphasis>, which is just a directory tree consisting of
 symlinks to the files of the active applications.  </para>
diff --git a/doc/manual/packages/profiles.xml b/doc/manual/packages/profiles.xml
index 4d10319ab..15085ab58 100644
--- a/doc/manual/packages/profiles.xml
+++ b/doc/manual/packages/profiles.xml
@@ -41,9 +41,9 @@ store.</para>
 $ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
 
 every time you want to run Subversion.  Of course we could set up the
-<envar>PATH</envar> environment variable to include the
+<literal>PATH</literal> environment variable to include the
 <filename>bin</filename> directory of every package we want to use,
-but this is not very convenient since changing <envar>PATH</envar>
+but this is not very convenient since changing <literal>PATH</literal>
 doesn’t take effect for already existing processes.  The solution Nix
 uses is to create directory trees of symlinks to
 <emphasis>activated</emphasis> packages.  These are called
@@ -122,10 +122,10 @@ $ nix-env --list-generations</screen></para>
 
 <para>You generally wouldn’t have
 <filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
-in your <envar>PATH</envar>.  Rather, there is a symlink
+in your <literal>PATH</literal>.  Rather, there is a symlink
 <filename>~/.nix-profile</filename> that points to your current
 profile.  This means that you should put
-<filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar>
+<filename>~/.nix-profile/bin</filename> in your <literal>PATH</literal>
 (and indeed, that’s what the initialisation script
 <filename>/nix/etc/profile.d/nix.sh</filename> does).  This makes it
 easier to switch to a different profile.  You can do that using the
diff --git a/doc/manual/release-notes/rl-0.10.xml b/doc/manual/release-notes/rl-0.10.xml
index 9afec4de9..9b4233546 100644
--- a/doc/manual/release-notes/rl-0.10.xml
+++ b/doc/manual/release-notes/rl-0.10.xml
@@ -184,9 +184,9 @@ irreversible.</para></warning>
 
 
   <listitem><para>Nix now works behind an HTTP proxy server; just set
-  the standard environment variables <envar>http_proxy</envar>,
-  <envar>https_proxy</envar>, <envar>ftp_proxy</envar> or
-  <envar>all_proxy</envar> appropriately.  Functions such as
+  the standard environment variables <literal>http_proxy</literal>,
+  <literal>https_proxy</literal>, <literal>ftp_proxy</literal> or
+  <literal>all_proxy</literal> appropriately.  Functions such as
   <function>fetchurl</function> in Nixpkgs also respect these
   variables.</para></listitem>
 
diff --git a/doc/manual/release-notes/rl-0.12.xml b/doc/manual/release-notes/rl-0.12.xml
index fdba8c4d5..18978ac4b 100644
--- a/doc/manual/release-notes/rl-0.12.xml
+++ b/doc/manual/release-notes/rl-0.12.xml
@@ -49,7 +49,7 @@ $ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</sc
   filesystem.  For instance, you can speed up an installation by
   mounting some remote Nix store that already has the packages in
   question via NFS or <literal>sshfs</literal>.  The environment
-  variable <envar>NIX_OTHER_STORES</envar> specifies the locations of
+  variable <literal>NIX_OTHER_STORES</literal> specifies the locations of
   the remote Nix directories,
   e.g. <literal>/mnt/remote-fs/nix</literal>.</para></listitem>
 
@@ -156,7 +156,7 @@ the following paths will be downloaded/copied (30.02 MiB):
 
   <listitem><para><command>nix-prefetch-url</command> now supports
   <literal>mirror://</literal> URLs, provided that the environment
-  variable <envar>NIXPKGS_ALL</envar> points at a Nixpkgs
+  variable <literal>NIXPKGS_ALL</literal> points at a Nixpkgs
   tree.</para></listitem>
 
   <listitem><para>Removed the commands
diff --git a/doc/manual/release-notes/rl-0.16.xml b/doc/manual/release-notes/rl-0.16.xml
index af1edc0eb..43b7622c6 100644
--- a/doc/manual/release-notes/rl-0.16.xml
+++ b/doc/manual/release-notes/rl-0.16.xml
@@ -30,7 +30,7 @@
       <replaceable>N</replaceable></option> as well as a configuration
       setting <varname>build-cores =
       <replaceable>N</replaceable></varname> that causes the
-      environment variable <envar>NIX_BUILD_CORES</envar> to be set to
+      environment variable <literal>NIX_BUILD_CORES</literal> to be set to
       <replaceable>N</replaceable> when the builder is invoked.  The
       builder can use this at its discretion to perform a parallel
       build, e.g., by calling <command>make -j
diff --git a/doc/manual/release-notes/rl-1.0.xml b/doc/manual/release-notes/rl-1.0.xml
index ff11168d0..41081a19f 100644
--- a/doc/manual/release-notes/rl-1.0.xml
+++ b/doc/manual/release-notes/rl-1.0.xml
@@ -29,7 +29,7 @@ previous release.  Here are the most significant:</para>
 
   <listitem>
     <para>Nix now has an search path for expressions.  The search path
-    is set using the environment variable <envar>NIX_PATH</envar> and
+    is set using the environment variable <literal>NIX_PATH</literal> and
     the <option>-I</option> command line option.  In Nix expressions,
     paths between angle brackets are used to specify files that must
     be looked up in the search path.  For instance, the expression
diff --git a/doc/manual/release-notes/rl-1.2.xml b/doc/manual/release-notes/rl-1.2.xml
index 748fd9e67..29bd5bf81 100644
--- a/doc/manual/release-notes/rl-1.2.xml
+++ b/doc/manual/release-notes/rl-1.2.xml
@@ -116,7 +116,7 @@ $ mount -o remount,ro,bind /nix/store
 
   <listitem>
     <para>Basic Nix expression evaluation profiling: setting the
-    environment variable <envar>NIX_COUNT_CALLS</envar> to
+    environment variable <literal>NIX_COUNT_CALLS</literal> to
     <literal>1</literal> will cause Nix to print how many times each
     primop or function was executed.</para>
   </listitem>
diff --git a/doc/manual/release-notes/rl-1.6.xml b/doc/manual/release-notes/rl-1.6.xml
index 580563420..cdb52ed0b 100644
--- a/doc/manual/release-notes/rl-1.6.xml
+++ b/doc/manual/release-notes/rl-1.6.xml
@@ -33,7 +33,7 @@ features:</para>
 
   <listitem>
     <para><command>nix-shell</command> now sets the shell prompt
-    (<envar>PS1</envar>) to ensure that Nix shells are distinguishable
+    (<literal>PS1</literal>) to ensure that Nix shells are distinguishable
     from your regular shells.</para>
   </listitem>
 
diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml
index c854c5c5f..326990774 100644
--- a/doc/manual/release-notes/rl-1.8.xml
+++ b/doc/manual/release-notes/rl-1.8.xml
@@ -105,7 +105,7 @@ $ nix-store -l $(which xterm)
   some terminals show them anyway.</para></listitem>
 
   <listitem><para>Various commands now automatically pipe their output
-  into the pager as specified by the <envar>PAGER</envar> environment
+  into the pager as specified by the <literal>PAGER</literal> environment
   variable.</para></listitem>
 
   <listitem><para>Several improvements to reduce memory consumption in
diff --git a/doc/manual/release-notes/rl-1.9.xml b/doc/manual/release-notes/rl-1.9.xml
index c8406bd20..cc6558b1a 100644
--- a/doc/manual/release-notes/rl-1.9.xml
+++ b/doc/manual/release-notes/rl-1.9.xml
@@ -62,7 +62,7 @@ $ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
       master branch.</para></listitem>
 
       <listitem><para>In the Nix search path (as specified via
-      <envar>NIX_PATH</envar> or <option>-I</option>). For example, to
+      <literal>NIX_PATH</literal> or <option>-I</option>). For example, to
       start a shell containing the Pan package from a specific version
       of Nixpkgs:
 
@@ -190,8 +190,8 @@ main = do
   “generation” symlink in <filename>/nix/var/nix/profiles</filename>
   if something actually changed.</para></listitem>
 
-  <listitem><para>The environment variable <envar>NIX_PAGER</envar>
-  can now be set to override <envar>PAGER</envar>. You can set it to
+  <listitem><para>The environment variable <literal>NIX_PAGER</literal>
+  can now be set to override <literal>PAGER</literal>. You can set it to
   <literal>cat</literal> to disable paging for Nix commands
   only.</para></listitem>
 
diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 4c683dd3d..57acfd0ba 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -30,7 +30,7 @@
   <listitem>
     <para>The “copy from other stores” substituter mechanism
     (<command>copy-from-other-stores</command> and the
-    <envar>NIX_OTHER_STORES</envar> environment variable) has been
+    <literal>NIX_OTHER_STORES</literal> environment variable) has been
     removed. It was primarily used by the NixOS installer to copy
     available paths from the installation medium. The replacement is
     to use a chroot store as a substituter
@@ -371,7 +371,7 @@
         daemon. You can use <literal>auto</literal> or the empty
         string to auto-select a local or daemon store depending on
         whether you have write permission to the Nix store. It is no
-        longer necessary to set the <envar>NIX_REMOTE</envar>
+        longer necessary to set the <literal>NIX_REMOTE</literal>
         environment variable to use the Nix daemon.</para>
 
         <para>As noted above, <classname>LocalStore</classname> now
@@ -492,7 +492,7 @@
         use <filename>/build</filename> instead of
         <filename>/tmp</filename> as the temporary build
         directory. This fixes potential security problems when a build
-        accidentally stores its <envar>TMPDIR</envar> in some
+        accidentally stores its <literal>TMPDIR</literal> in some
         security-sensitive place, such as an RPATH.</para>
       </listitem>
 
@@ -546,8 +546,8 @@
     are not using the Nix daemon, you can now just specify a remote
     build machine on the command line, e.g. <literal>--option builders
     'ssh://my-mac x86_64-darwin'</literal>. The environment variable
-    <envar>NIX_BUILD_HOOK</envar> has been removed and is no longer
-    needed. The environment variable <envar>NIX_REMOTE_SYSTEMS</envar>
+    <literal>NIX_BUILD_HOOK</literal> has been removed and is no longer
+    needed. The environment variable <literal>NIX_REMOTE_SYSTEMS</literal>
     is still supported for compatibility, but it is also possible to
     specify builders in <command>nix.conf</command> by setting the
     option <literal>builders =
@@ -574,7 +574,7 @@
   </listitem>
 
   <listitem>
-    <para><envar>NIX_PATH</envar> is now lazy, so URIs in the path are
+    <para><literal>NIX_PATH</literal> is now lazy, so URIs in the path are
     only downloaded if they are needed for evaluation.</para>
   </listitem>
 
@@ -672,7 +672,7 @@
     <para><command>nix-shell</command> now uses
     <varname>bashInteractive</varname> from Nixpkgs, rather than the
     <command>bash</command> command that happens to be in the caller’s
-    <envar>PATH</envar>. This is especially important on macOS where
+    <literal>PATH</literal>. This is especially important on macOS where
     the <command>bash</command> provided by the system is seriously
     outdated and cannot execute <literal>stdenv</literal>’s setup
     script.</para>
@@ -720,7 +720,7 @@
 configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";
 </programlisting>
 
-    will cause the <envar>configureFlags</envar> environment variable
+    will cause the <literal>configureFlags</literal> environment variable
     to contain the actual store paths corresponding to the
     <literal>out</literal> and <literal>dev</literal> outputs.</para>
   </listitem>
@@ -815,7 +815,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     <varname>__structuredAttrs</varname> is set to
     <literal>true</literal>, the other derivation attributes are
     serialised in JSON format and made available to the builder via
-    the file <envar>.attrs.json</envar> in the builder’s temporary
+    the file <literal>.attrs.json</literal> in the builder’s temporary
     directory. This obviates the need for
     <varname>passAsFile</varname> since JSON files have no size
     restrictions, unlike process environments.</para>
@@ -823,7 +823,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     <para><link
     xlink:href="https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699">As
     a convenience to Bash builders</link>, Nix writes a script named
-    <envar>.attrs.sh</envar> to the builder’s directory that
+    <literal>.attrs.sh</literal> to the builder’s directory that
     initialises shell variables corresponding to all attributes that
     are representable in Bash. This includes non-nested (associative)
     arrays. For example, the attribute <literal>hardening.format =
@@ -835,7 +835,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     <para>Builders can <link
     xlink:href="https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3">now</link>
     communicate what build phase they are in by writing messages to
-    the file descriptor specified in <envar>NIX_LOG_FD</envar>. The
+    the file descriptor specified in <literal>NIX_LOG_FD</literal>. The
     current phase is shown by the <command>nix</command> progress
     indicator.
     </para>
diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml
index d29eb87e8..cc992fabb 100644
--- a/doc/manual/release-notes/rl-2.2.xml
+++ b/doc/manual/release-notes/rl-2.2.xml
@@ -114,8 +114,8 @@ import &lt;nix/fetchurl.nix> {
 
   <listitem>
     <para>The evaluator now prints profiling statistics (enabled via
-    the <envar>NIX_SHOW_STATS</envar> and
-    <envar>NIX_COUNT_CALLS</envar> environment variables) in JSON
+    the <literal>NIX_SHOW_STATS</literal> and
+    <literal>NIX_COUNT_CALLS</literal> environment variables) in JSON
     format.</para>
   </listitem>
 
diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
index 0ad7d641f..83accf33e 100644
--- a/doc/manual/release-notes/rl-2.3.xml
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -60,7 +60,7 @@ incompatible changes:</para>
 
   <listitem>
     <para>Builds are now executed in a pseudo-terminal, and the
-    <envar>TERM</envar> environment variable is set to
+    <literal>TERM</literal> environment variable is set to
     <literal>xterm-256color</literal>. This allows many programs
     (e.g. <command>gcc</command>, <command>clang</command>,
     <command>cmake</command>) to print colorized log output.</para>

From f3903035667e158112dfd414091d8d50ef90c5f4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 10:44:54 +0200
Subject: [PATCH 026/882] Reconvert

---
 doc/manual/src/SUMMARY.md                     |   2 +
 .../src/advanced-topics/cores-vs-jobs.md      |  10 +-
 .../src/advanced-topics/distributed-builds.md |   2 +-
 doc/manual/src/command-ref/command-ref.md     |   2 +
 doc/manual/src/command-ref/env-common.md      | 112 +++++++++
 doc/manual/src/command-ref/opt-common.md      | 221 ++++++++++++++++++
 .../src/expressions/advanced-attributes.md    |  14 +-
 doc/manual/src/expressions/build-script.md    |  20 +-
 doc/manual/src/expressions/builder-syntax.md  |  20 +-
 doc/manual/src/expressions/derivations.md     |  16 +-
 doc/manual/src/expressions/generic-builder.md |  13 +-
 doc/manual/src/expressions/language-values.md |   4 +-
 .../expressions/simple-building-testing.md    |   2 +-
 doc/manual/src/installation/env-variables.md  |  10 +-
 .../src/installation/installing-binary.md     |   4 +-
 doc/manual/src/installation/multi-user.md     |   2 +-
 .../package-management/basic-package-mgmt.md  |   8 +-
 doc/manual/src/package-management/profiles.md |   8 +-
 doc/manual/src/release-notes/rl-0.10.md       |   4 +-
 doc/manual/src/release-notes/rl-0.12.md       |   4 +-
 doc/manual/src/release-notes/rl-0.16.md       |   2 +-
 doc/manual/src/release-notes/rl-1.0.md        |   2 +-
 doc/manual/src/release-notes/rl-1.2.md        |   2 +-
 doc/manual/src/release-notes/rl-1.6.md        |   2 +-
 doc/manual/src/release-notes/rl-1.8.md        |   2 +-
 doc/manual/src/release-notes/rl-1.9.md        |   8 +-
 doc/manual/src/release-notes/rl-2.0.md        |  28 +--
 doc/manual/src/release-notes/rl-2.2.md        |   2 +-
 doc/manual/src/release-notes/rl-2.3.md        |   2 +-
 29 files changed, 433 insertions(+), 95 deletions(-)
 create mode 100644 doc/manual/src/command-ref/env-common.md
 create mode 100644 doc/manual/src/command-ref/opt-common.md

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index f562d1db4..f9ef1b060 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -45,6 +45,8 @@
   - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
   - [Using the `post-build-hook`](advanced-topics/post-build-hook.md)
 - [Command Reference](command-ref/command-ref.md)
+  - [Common Options](command-ref/opt-common.md)
+  - [Common Environment Variables](command-ref/env-common.md)
   - [Utilities](command-ref/utilities.md)
     - [nix-copy-closure](command-ref/nix-copy-closure.md)
 - [Glossary](glossary.md)
diff --git a/doc/manual/src/advanced-topics/cores-vs-jobs.md b/doc/manual/src/advanced-topics/cores-vs-jobs.md
index 846b6356e..91e95d5b4 100644
--- a/doc/manual/src/advanced-topics/cores-vs-jobs.md
+++ b/doc/manual/src/advanced-topics/cores-vs-jobs.md
@@ -16,18 +16,18 @@ configuration trade-offs.
     -j`.
 
 The [???](#conf-cores) setting determines the value of
-NIX\_BUILD\_CORES. NIX\_BUILD\_CORES is equal to [???](#conf-cores),
-unless [???](#conf-cores) equals `0`, in which case NIX\_BUILD\_CORES
+`NIX_BUILD_CORES`. `NIX_BUILD_CORES` is equal to [???](#conf-cores),
+unless [???](#conf-cores) equals `0`, in which case `NIX_BUILD_CORES`
 will be the total number of cores in the system.
 
 The maximum number of consumed cores is a simple multiplication,
-[???](#conf-max-jobs) \* NIX\_BUILD\_CORES.
+[???](#conf-max-jobs) \* `NIX_BUILD_CORES`.
 
 The balance on how to set these two independent variables depends upon
 each builder's workload and hardware. Here are a few example scenarios
 on a machine with 24 cores:
 
-| [???](#conf-max-jobs) | [???](#conf-cores) | NIX\_BUILD\_CORES | Maximum Processes | Result                                                                                                                                                                                                                                                                                 |
+| [???](#conf-max-jobs) | [???](#conf-cores) | `NIX_BUILD_CORES` | Maximum Processes | Result                                                                                                                                                                                                                                                                                 |
 | --------------------- | ------------------ | ----------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | 1                     | 24                 | 24                | 24                | One derivation will be built at a time, each one can use 24 cores. Undersold if a job can’t use 24 cores.                                                                                                                                                                              |
 | 4                     | 6                  | 6                 | 24                | Four derivations will be built at once, each given access to six cores.                                                                                                                                                                                                                |
@@ -38,5 +38,5 @@ on a machine with 24 cores:
 Balancing 24 Build Cores
 
 It is up to the derivations' build script to respect host's requested
-cores-per-build by following the value of the NIX\_BUILD\_CORES
+cores-per-build by following the value of the `NIX_BUILD_CORES`
 environment variable.
diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
index 197708ee5..4f24febb0 100644
--- a/doc/manual/src/advanced-topics/distributed-builds.md
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -27,7 +27,7 @@ If you get the error
     bash: nix-store: command not found
     error: cannot connect to 'mac'
 
-then you need to ensure that the PATH of non-interactive login shells
+then you need to ensure that the `PATH` of non-interactive login shells
 contains Nix.
 
 > **Warning**
diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/command-ref.md
index b15a50a3b..2c7dbf22e 100644
--- a/doc/manual/src/command-ref/command-ref.md
+++ b/doc/manual/src/command-ref/command-ref.md
@@ -1,2 +1,4 @@
+# Command Reference
+
 This section lists commands and options that you can use when you
 work with Nix.
diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
new file mode 100644
index 000000000..14e08e0f1
--- /dev/null
+++ b/doc/manual/src/command-ref/env-common.md
@@ -0,0 +1,112 @@
+# Common Environment Variables
+
+Most Nix commands interpret the following environment variables:
+
+  - `IN_NIX_SHELL`  
+    Indicator that tells if the current environment was set up by
+    `nix-shell`. Since Nix 2.0 the values are `"pure"` and `"impure"`
+
+  - `NIX_PATH`  
+    A colon-separated list of directories used to look up Nix
+    expressions enclosed in angle brackets (i.e., `<path>`). For
+    instance, the value
+    
+        /home/eelco/Dev:/etc/nixos
+    
+    will cause Nix to look for paths relative to `/home/eelco/Dev` and
+    `/etc/nixos`, in this order. It is also possible to match paths
+    against a prefix. For example, the value
+    
+        nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
+    
+    will cause Nix to search for `<nixpkgs/path>` in
+    `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
+    
+    If a path in the Nix search path starts with `http://` or
+    `https://`, it is interpreted as the URL of a tarball that will be
+    downloaded and unpacked to a temporary location. The tarball must
+    consist of a single top-level directory. For example, setting
+    `NIX_PATH` to
+    
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
+    
+    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
+    channel.
+    
+    A following shorthand can be used to refer to the official channels:
+    
+        nixpkgs=channel:nixos-15.09
+    
+    The search path can be extended using the `-I` option, which takes
+    precedence over `NIX_PATH`.
+
+  - `NIX_IGNORE_SYMLINK_STORE`  
+    Normally, the Nix store directory (typically `/nix/store`) is not
+    allowed to contain any symlink components. This is to prevent
+    “impure” builds. Builders sometimes “canonicalise” paths by
+    resolving all symlink components. Thus, builds on different machines
+    (with `/nix/store` resolving to different locations) could yield
+    different results. This is generally not a problem, except when
+    builds are deployed to machines where `/nix/store` resolves
+    differently. If you are sure that you’re not going to do that, you
+    can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
+    
+    Note that if you’re symlinking the Nix store so that you can put it
+    on another file system than the root file system, on Linux you’re
+    better off using `bind` mount points, e.g.,
+    
+        $ mkdir /nix
+        $ mount -o bind /mnt/otherdisk/nix /nix
+    
+    Consult the mount 8 manual page for details.
+
+  - `NIX_STORE_DIR`  
+    Overrides the location of the Nix store (default `prefix/store`).
+
+  - `NIX_DATA_DIR`  
+    Overrides the location of the Nix static data directory (default
+    `prefix/share`).
+
+  - `NIX_LOG_DIR`  
+    Overrides the location of the Nix log directory (default
+    `prefix/var/log/nix`).
+
+  - `NIX_STATE_DIR`  
+    Overrides the location of the Nix state directory (default
+    `prefix/var/nix`).
+
+  - `NIX_CONF_DIR`  
+    Overrides the location of the system Nix configuration directory
+    (default `prefix/etc/nix`).
+
+  - `NIX_USER_CONF_FILES`  
+    Overrides the location of the user Nix configuration files to load
+    from (defaults to the XDG spec locations). The variable is treated
+    as a list separated by the `:` token.
+
+  - `TMPDIR`  
+    Use the specified directory to store temporary files. In particular,
+    this includes temporary build directories; these can take up
+    substantial amounts of disk space. The default is `/tmp`.
+
+  - `NIX_REMOTE`  
+    This variable should be set to `daemon` if you want to use the Nix
+    daemon to execute Nix operations. This is necessary in [multi-user
+    Nix installations](#ssec-multi-user). If the Nix daemon's Unix
+    socket is at some non-standard path, this variable should be set to
+    `unix://path/to/socket`. Otherwise, it should be left unset.
+
+  - `NIX_SHOW_STATS`  
+    If set to `1`, Nix will print some evaluation statistics, such as
+    the number of values allocated.
+
+  - `NIX_COUNT_CALLS`  
+    If set to `1`, Nix will print how often functions were called during
+    Nix expression evaluation. This is useful for profiling your Nix
+    expressions.
+
+  - `GC_INITIAL_HEAP_SIZE`  
+    If Nix has been configured to use the Boehm garbage collector, this
+    variable sets the initial size of the heap in bytes. It defaults to
+    384 MiB. Setting it to a low value reduces memory consumption, but
+    will increase runtime due to the overhead of garbage collection.
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
new file mode 100644
index 000000000..ac9d996c3
--- /dev/null
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -0,0 +1,221 @@
+# Common Options
+
+Most Nix commands accept the following command-line options:
+
+  - `--help`  
+    Prints out a summary of the command syntax and exits.
+
+  - `--version`  
+    Prints out the Nix version number on standard output and exits.
+
+  - `--verbose` / `-v`  
+    Increases the level of verbosity of diagnostic messages printed on
+    standard error. For each Nix operation, the information printed on
+    standard output is well-defined; any diagnostic information is
+    printed on standard error, never on standard output.
+    
+    This option may be specified repeatedly. Currently, the following
+    verbosity levels exist:
+    
+      - 0  
+        “Errors only”: only print messages explaining why the Nix
+        invocation failed.
+    
+      - 1  
+        “Informational”: print *useful* messages about what Nix is
+        doing. This is the default.
+    
+      - 2  
+        “Talkative”: print more informational messages.
+    
+      - 3  
+        “Chatty”: print even more informational messages.
+    
+      - 4  
+        “Debug”: print debug information.
+    
+      - 5  
+        “Vomit”: print vast amounts of debug information.
+
+  - `--quiet`  
+    Decreases the level of verbosity of diagnostic messages printed on
+    standard error. This is the inverse option to `-v` / `--verbose`.
+    
+    This option may be specified repeatedly. See the previous verbosity
+    levels list.
+
+  - `--log-format` format  
+    This option can be used to change the output of the log format, with
+    format being one of:
+    
+      - raw  
+        This is the raw format, as outputted by nix-build.
+    
+      - internal-json  
+        Outputs the logs in a structured manner. NOTE: the json schema
+        is not guarantees to be stable between releases.
+    
+      - bar  
+        Only display a progress bar during the builds.
+    
+      - bar-with-logs  
+        Display the raw logs, with the progress bar at the bottom.
+
+  - `--no-build-output` / `-Q`  
+    By default, output written by builders to standard output and
+    standard error is echoed to the Nix command's standard error. This
+    option suppresses this behaviour. Note that the builder's standard
+    output and error are always written to a log file in
+    `prefix/nix/var/log/nix`.
+
+  - `--max-jobs` / `-j` number  
+    Sets the maximum number of build jobs that Nix will perform in
+    parallel to the specified number. Specify `auto` to use the number
+    of CPUs in the system. The default is specified by the
+    [`max-jobs`](#conf-max-jobs) configuration setting, which itself
+    defaults to `1`. A higher value is useful on SMP systems or to
+    exploit I/O latency.
+    
+    Setting it to `0` disallows building on the local machine, which is
+    useful when you want builds to happen only on remote builders.
+
+  - `--cores`  
+    Sets the value of the `NIX_BUILD_CORES` environment variable in the
+    invocation of builders. Builders can use this variable at their
+    discretion to control the maximum amount of parallelism. For
+    instance, in Nixpkgs, if the derivation attribute
+    `enableParallelBuilding` is set to `true`, the builder passes the
+    `-jN` flag to GNU Make. It defaults to the value of the
+    [`cores`](#conf-cores) configuration setting, if set, or `1`
+    otherwise. The value `0` means that the builder should use all
+    available CPU cores in the system.
+
+  - `--max-silent-time`  
+    Sets the maximum number of seconds that a builder can go without
+    producing any data on standard output or standard error. The default
+    is specified by the [`max-silent-time`](#conf-max-silent-time)
+    configuration setting. `0` means no time-out.
+
+  - `--timeout`  
+    Sets the maximum number of seconds that a builder can run. The
+    default is specified by the [`timeout`](#conf-timeout) configuration
+    setting. `0` means no timeout.
+
+  - `--keep-going` / `-k`  
+    Keep going in case of failed builds, to the greatest extent
+    possible. That is, if building an input of some derivation fails,
+    Nix will still build the other inputs, but not the derivation
+    itself. Without this option, Nix stops if any build fails (except
+    for builds of substitutes), possibly killing builds in progress (in
+    case of parallel or distributed builds).
+
+  - `--keep-failed` / `-K`  
+    Specifies that in case of a build failure, the temporary directory
+    (usually in `/tmp`) in which the build takes place should not be
+    deleted. The path of the build directory is printed as an
+    informational message.
+
+  - `--fallback`  
+    Whenever Nix attempts to build a derivation for which substitutes
+    are known for each output path, but realising the output paths
+    through the substitutes fails, fall back on building the derivation.
+    
+    The most common scenario in which this is useful is when we have
+    registered substitutes in order to perform binary distribution from,
+    say, a network repository. If the repository is down, the
+    realisation of the derivation will fail. When this option is
+    specified, Nix will build the derivation instead. Thus, installation
+    from binaries falls back on installation from source. This option is
+    not the default since it is generally not desirable for a transient
+    failure in obtaining the substitutes to lead to a full build from
+    source (with the related consumption of resources).
+
+  - `--no-build-hook`  
+    Disables the build hook mechanism. This allows to ignore remote
+    builders if they are setup on the machine.
+    
+    It's useful in cases where the bandwidth between the client and the
+    remote builder is too low. In that case it can take more time to
+    upload the sources to the remote builder and fetch back the result
+    than to do the computation locally.
+
+  - `--readonly-mode`  
+    When this option is used, no attempt is made to open the Nix
+    database. Most Nix operations do need database access, so those
+    operations will fail.
+
+  - `--arg` name value  
+    This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell`
+    and `nix-build`. When evaluating Nix expressions, the expression
+    evaluator will automatically try to call functions that it
+    encounters. It can automatically call functions for which every
+    argument has a [default value](#ss-functions) (e.g., `{ argName ?
+            defaultValue }:
+            ...`). With `--arg`, you can also call functions that have arguments
+    without a default value (or override a default value). That is, if
+    the evaluator encounters a function with an argument named name, it
+    will call it with value value.
+    
+    For instance, the top-level `default.nix` in Nixpkgs is actually a
+    function:
+    
+        { # The system (e.g., `i686-linux') for which to build the packages.
+          system ? builtins.currentSystem
+          ...
+        }: ...
+    
+    So if you call this Nix expression (e.g., when you do `nix-env -i
+    pkgname`), the function will be called automatically using the value
+    [`builtins.currentSystem`](#builtin-currentSystem) for the `system`
+    argument. You can override this using `--arg`, e.g., `nix-env -i
+            pkgname --arg system
+            \"i686-freebsd\"`. (Note that since the argument is a Nix string
+    literal, you have to escape the quotes.)
+
+  - `--argstr` name value  
+    This option is like `--arg`, only the value is not a Nix expression
+    but a string. So instead of `--arg system \"i686-linux\"` (the outer
+    quotes are to keep the shell happy) you can say `--argstr system
+            i686-linux`.
+
+  - `--attr` / `-A` attrPath  
+    Select an attribute from the top-level Nix expression being
+    evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and
+    `nix-shell` only.) The *attribute path* attrPath is a sequence of
+    attribute names separated by dots. For instance, given a top-level
+    Nix expression e, the attribute path `xorg.xorgserver` would cause
+    the expression `e.xorg.xorgserver` to be used. See [`nix-env
+            --install`](#refsec-nix-env-install-examples) for some concrete
+    examples.
+    
+    In addition to attribute names, you can also specify array indices.
+    For instance, the attribute path `foo.3.bar` selects the `bar`
+    attribute of the fourth element of the array in the `foo` attribute
+    of the top-level expression.
+
+  - `--expr` / `-E`  
+    Interpret the command line arguments as a list of Nix expressions to
+    be parsed and evaluated, rather than as a list of file names of Nix
+    expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.)
+    
+    For `nix-shell`, this option is commonly used to give you a shell in
+    which you can build the packages returned by the expression. If you
+    want to get a shell which contain the *built* packages ready for
+    use, give your expression to the `nix-shell -p` convenience flag
+    instead.
+
+  - `-I` path  
+    Add a path to the Nix expression search path. This option may be
+    given multiple times. See the NIX\_PATH\</literal\> environment
+    variable for information on the semantics of the Nix search path.
+    Paths added through `-I` take precedence over `NIX_PATH`.
+
+  - `--option` name value  
+    Set the Nix configuration option name to value. This overrides
+    settings in the Nix configuration file (see nix.conf5).
+
+  - `--repair`  
+    Fix corrupted or missing store paths by redownloading or rebuilding
+    them. Note that this is slow because it requires computing a
+    cryptographic hash of the contents of every path in the closure of
+    the build. Also note the warning under `nix-store --repair-path`.
diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
index 683b504a7..01e18513d 100644
--- a/doc/manual/src/expressions/advanced-attributes.md
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -87,7 +87,7 @@ Derivations can declare some infrequently used optional attributes.
         impureEnvVars = [ "http_proxy" "https_proxy" ... ];
     
     to make it use the proxy server configuration specified by the user
-    in the environment variables http\_proxy and friends.
+    in the environment variables `http_proxy` and friends.
     
     This attribute is only allowed in [fixed-output
     derivations](#fixed-output-drvs), where impurities such as these are
@@ -201,15 +201,15 @@ Derivations can declare some infrequently used optional attributes.
         
     ```
     
-    then when the builder runs, the environment variable bigPath will
+    then when the builder runs, the environment variable `bigPath` will
     contain the absolute path to a temporary file containing `a very
     long
                     string`. That is, for any attribute x listed in `passAsFile`, Nix
-    will pass an environment variable xPath holding the path of the file
-    containing the value of attribute x. This is useful when you need to
-    pass large strings to a builder, since most operating systems impose
-    a limit on the size of the environment (typically, a few hundred
-    kilobyte).
+    will pass an environment variable `xPath` holding the path of the
+    file containing the value of attribute x. This is useful when you
+    need to pass large strings to a builder, since most operating
+    systems impose a limit on the size of the environment (typically, a
+    few hundred kilobyte).
 
   - `preferLocalBuild`  
     If this attribute is set to `true` and [distributed building is
diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md
index 256a5cd44..a359ebc46 100644
--- a/doc/manual/src/expressions/build-script.md
+++ b/doc/manual/src/expressions/build-script.md
@@ -19,29 +19,29 @@ what a builder does. It performs the following steps:
 
   - When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the derivation).
-    For instance, the PATH variable is empty\[1\]. This is done to
+    For instance, the `PATH` variable is empty\[1\]. This is done to
     prevent undeclared inputs from being used in the build process. If
-    for example the PATH contained `/usr/bin`, then you might
+    for example the `PATH` contained `/usr/bin`, then you might
     accidentally use `/usr/bin/gcc`.
     
     So the first step is to set up the environment. This is done by
     calling the `setup` script of the standard environment. The
-    environment variable stdenv points to the location of the standard
+    environment variable `stdenv` points to the location of the standard
     environment being used. (It wasn't specified explicitly as an
     attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
     automatically.)
 
   - Since Hello needs Perl, we have to make sure that Perl is in the
-    PATH. The perl environment variable points to the location of the
-    Perl package (since it was passed in as an attribute to the
+    `PATH`. The `perl` environment variable points to the location of
+    the Perl package (since it was passed in as an attribute to the
     derivation), so `$perl/bin` is the directory containing the Perl
     interpreter.
 
   - Now we have to unpack the sources. The `src` attribute was bound to
     the result of fetching the Hello source tarball from the network, so
-    the src environment variable points to the location in the Nix store
-    to which the tarball was downloaded. After unpacking, we `cd` to the
-    resulting source directory.
+    the `src` environment variable points to the location in the Nix
+    store to which the tarball was downloaded. After unpacking, we `cd`
+    to the resulting source directory.
     
     The whole build is performed in a temporary directory created in
     `/tmp`, by the way. This directory is removed after the builder
@@ -55,13 +55,13 @@ what a builder does. It performs the following steps:
     separate location in the Nix store, for instance
     `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix
     computes this path by cryptographically hashing all attributes of
-    the derivation. The path is passed to the builder through the out
+    the derivation. The path is passed to the builder through the `out`
     environment variable. So here we give `configure` the parameter
     `--prefix=$out` to cause Hello to be installed in the expected
     location.
 
   - Finally we build Hello (`make`) and install it into the location
-    specified by out (`make install`).
+    specified by `out` (`make install`).
 
 If you are wondering about the absence of error checking on the result
 of various commands called in the builder: this is because the shell
diff --git a/doc/manual/src/expressions/builder-syntax.md b/doc/manual/src/expressions/builder-syntax.md
index c92acf106..916159d40 100644
--- a/doc/manual/src/expressions/builder-syntax.md
+++ b/doc/manual/src/expressions/builder-syntax.md
@@ -19,29 +19,29 @@ what a builder does. It performs the following steps:
 
   - When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the derivation).
-    For instance, the PATH variable is empty\[1\]. This is done to
+    For instance, the `PATH` variable is empty\[1\]. This is done to
     prevent undeclared inputs from being used in the build process. If
-    for example the PATH contained `/usr/bin`, then you might
+    for example the `PATH` contained `/usr/bin`, then you might
     accidentally use `/usr/bin/gcc`.
     
     So the first step is to set up the environment. This is done by
     calling the `setup` script of the standard environment. The
-    environment variable stdenv points to the location of the standard
+    environment variable `stdenv` points to the location of the standard
     environment being used. (It wasn't specified explicitly as an
     attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
     automatically.)
 
   - Since Hello needs Perl, we have to make sure that Perl is in the
-    PATH. The perl environment variable points to the location of the
-    Perl package (since it was passed in as an attribute to the
+    `PATH`. The `perl` environment variable points to the location of
+    the Perl package (since it was passed in as an attribute to the
     derivation), so `$perl/bin` is the directory containing the Perl
     interpreter.
 
   - Now we have to unpack the sources. The `src` attribute was bound to
     the result of fetching the Hello source tarball from the network, so
-    the src environment variable points to the location in the Nix store
-    to which the tarball was downloaded. After unpacking, we `cd` to the
-    resulting source directory.
+    the `src` environment variable points to the location in the Nix
+    store to which the tarball was downloaded. After unpacking, we `cd`
+    to the resulting source directory.
     
     The whole build is performed in a temporary directory created in
     `/tmp`, by the way. This directory is removed after the builder
@@ -55,13 +55,13 @@ what a builder does. It performs the following steps:
     separate location in the Nix store, for instance
     `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix
     computes this path by cryptographically hashing all attributes of
-    the derivation. The path is passed to the builder through the out
+    the derivation. The path is passed to the builder through the `out`
     environment variable. So here we give `configure` the parameter
     `--prefix=$out` to cause Hello to be installed in the expected
     location.
 
   - Finally we build Hello (`make`) and install it into the location
-    specified by out (`make install`).
+    specified by `out` (`make install`).
 
 If you are wondering about the absence of error checking on the result
 of various commands called in the builder: this is because the shell
diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/expressions/derivations.md
index 7ffc6fabe..a4df20baa 100644
--- a/doc/manual/src/expressions/derivations.md
+++ b/doc/manual/src/expressions/derivations.md
@@ -83,32 +83,32 @@ as a command-line argument. See the Nixpkgs manual for details.
 The builder is executed as follows:
 
   - A temporary directory is created under the directory specified by
-    TMPDIR (default `/tmp`) where the build will take place. The current
-    directory is changed to this directory.
+    `TMPDIR` (default `/tmp`) where the build will take place. The
+    current directory is changed to this directory.
 
   - The environment is cleared and set to the derivation attributes, as
     specified above.
 
   - In addition, the following variables are set:
     
-      - NIX\_BUILD\_TOP contains the path of the temporary directory for
+      - `NIX_BUILD_TOP` contains the path of the temporary directory for
         this build.
     
-      - Also, TMPDIR, TEMPDIR, TMP, TEMP are set to point to the
+      - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the
         temporary directory. This is to prevent the builder from
         accidentally writing temporary files anywhere else. Doing so
         might cause interference by other processes.
     
-      - PATH is set to `/path-not-set` to prevent shells from
+      - `PATH` is set to `/path-not-set` to prevent shells from
         initialising it to their built-in default value.
     
-      - HOME is set to `/homeless-shelter` to prevent programs from
+      - `HOME` is set to `/homeless-shelter` to prevent programs from
         using `/etc/passwd` or the like to find the user's home
-        directory, which could cause impurity. Usually, when HOME is
+        directory, which could cause impurity. Usually, when `HOME` is
         set, it is used as the location of the home directory, even if
         it points to a non-existent path.
     
-      - NIX\_STORE is set to the path of the top-level Nix store
+      - `NIX_STORE` is set to the path of the top-level Nix store
         directory (typically, `/nix/store`).
     
       - For each output declared in `outputs`, the corresponding
diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md
index cbc484199..a00b08b55 100644
--- a/doc/manual/src/expressions/generic-builder.md
+++ b/doc/manual/src/expressions/generic-builder.md
@@ -22,9 +22,9 @@ build facilities in shown in [example\_title](#ex-hello-builder2).
     
     genericBuild 
 
-  - The buildInputs variable tells `setup` to use the indicated packages
-    as “inputs”. This means that if a package provides a `bin`
-    subdirectory, it's added to PATH; if it has a `include`
+  - The `buildInputs` variable tells `setup` to use the indicated
+    packages as “inputs”. This means that if a package provides a `bin`
+    subdirectory, it's added to `PATH`; if it has a `include`
     subdirectory, it's added to GCC's header search path; and so
     on.\[1\]
 
@@ -37,7 +37,7 @@ build facilities in shown in [example\_title](#ex-hello-builder2).
     It can be customised in many ways; see the Nixpkgs manual for
     details.
 
-Discerning readers will note that the buildInputs could just as well
+Discerning readers will note that the `buildInputs` could just as well
 have been set in the Nix expression, like this:
 
 ``` 
@@ -57,5 +57,6 @@ entirely.
 1.  How does it work? `setup` tries to source the file
     `pkg/nix-support/setup-hook` of all dependencies. These “setup
     hooks” can then set up whatever environment variables they want;
-    for instance, the setup hook for Perl sets the PERL5LIB environment
-    variable to contain the `lib/site_perl` directories of all inputs.
+    for instance, the setup hook for Perl sets the `PERL5LIB`
+    environment variable to contain the `lib/site_perl` directories of
+    all inputs.
diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md
index 98ad97c97..fac0be47e 100644
--- a/doc/manual/src/expressions/language-values.md
+++ b/doc/manual/src/expressions/language-values.md
@@ -120,8 +120,8 @@ Nix has the following basic data types:
     
     Paths can also be specified between angle brackets, e.g.
     `<nixpkgs>`. This means that the directories listed in the
-    environment variable NIX\_PATH will be searched for the given file
-    or directory name.
+    environment variable NIX\_PATH\</literal\> will be searched for the
+    given file or directory name.
 
   - *Booleans* with values `true` and `false`.
 
diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md
index bc064c733..c3e8d7f82 100644
--- a/doc/manual/src/expressions/simple-building-testing.md
+++ b/doc/manual/src/expressions/simple-building-testing.md
@@ -32,7 +32,7 @@ of the build will be safely kept on your system. You can use
 
 Nix has transactional semantics. Once a build finishes successfully, Nix
 makes a note of this in its database: it registers that the path denoted
-by out is now “valid”. If you try to build the derivation again, Nix
+by `out` is now “valid”. If you try to build the derivation again, Nix
 will see that the path is already valid and finish immediately. If a
 build fails, either because it returns a non-zero exit code, because Nix
 or the builder are killed, or because the machine crashes, then the
diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md
index 7946ac437..6e78245c9 100644
--- a/doc/manual/src/installation/env-variables.md
+++ b/doc/manual/src/installation/env-variables.md
@@ -1,7 +1,7 @@
 # Environment Variables
 
 To use Nix, some environment variables should be set. In particular,
-PATH should contain the directories `prefix/bin` and
+`PATH` should contain the directories `prefix/bin` and
 `~/.nix-profile/bin`. The first directory contains the Nix tools
 themselves, while `~/.nix-profile` is a symbolic link to the current
 *user environment* (an automatically generated package consisting of
@@ -12,13 +12,13 @@ this:
 
     source prefix/etc/profile.d/nix.sh
 
-# NIX\_SSL\_CERT\_FILE
+# `NIX_SSL_CERT_FILE`
 
 If you need to specify a custom certificate bundle to account for an
 HTTPS-intercepting man in the middle proxy, you must specify the path to
-the certificate bundle in the environment variable NIX\_SSL\_CERT\_FILE.
+the certificate bundle in the environment variable `NIX_SSL_CERT_FILE`.
 
-If you don't specify a NIX\_SSL\_CERT\_FILE manually, Nix will install
+If you don't specify a `NIX_SSL_CERT_FILE` manually, Nix will install
 and use its own certificate bundle.
 
 Set the environment variable and install Nix
@@ -36,7 +36,7 @@ In the shell profile and rc files (for example, `/etc/bashrc`,
 > You must not add the export and then do the install, as the Nix
 > installer will detect the presense of Nix configuration, and abort.
 
-## NIX\_SSL\_CERT\_FILE with macOS and the Nix daemon
+## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon
 
 On macOS you must specify the environment variable for the Nix daemon
 service, then restart it:
diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index d4e412e67..6983452d3 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -34,8 +34,8 @@ manually create `/nix` first as root, e.g.:
 The install script will modify the first writable file from amongst
 `.bash_profile`, `.bash_login` and `.profile` to source
 `~/.nix-profile/etc/profile.d/nix.sh`. You can set the
-NIX\_INSTALLER\_NO\_MODIFY\_PROFILE environment variable before
-executing the install script to disable this behaviour.
+`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing
+the install script to disable this behaviour.
 
 You can uninstall Nix simply by running:
 
diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/src/installation/multi-user.md
index 17286fdc5..6493e717b 100644
--- a/doc/manual/src/installation/multi-user.md
+++ b/doc/manual/src/installation/multi-user.md
@@ -43,7 +43,7 @@ The [Nix daemon](#sec-nix-daemon) should be started as follows (as
 You’ll want to put that line somewhere in your system’s boot scripts.
 
 To let unprivileged users use the daemon, they should set the
-[NIX\_REMOTE environment variable](#envar-remote) to `daemon`. So you
+[`NIX_REMOTE` environment variable](#envar-remote) to `daemon`. So you
 should put a line like
 
     export NIX_REMOTE=daemon
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
index f5c550fd7..04dad3339 100644
--- a/doc/manual/src/package-management/basic-package-mgmt.md
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -8,10 +8,10 @@ In Nix, different users can have different “views” on the set of
 installed applications. That is, there might be lots of applications
 present on the system (possibly in many different versions), but users
 can have a specific selection of those active — where “active” just
-means that it appears in a directory in the user’s PATH. Such a view on
-the set of installed applications is called a *user environment*, which
-is just a directory tree consisting of symlinks to the files of the
-active applications.
+means that it appears in a directory in the user’s `PATH`. Such a view
+on the set of installed applications is called a *user environment*,
+which is just a directory tree consisting of symlinks to the files of
+the active applications.
 
 Components are installed from a set of *Nix expressions* that tell Nix
 how to build those packages, including, if necessary, their
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
index c46adf538..984afca55 100644
--- a/doc/manual/src/package-management/profiles.md
+++ b/doc/manual/src/package-management/profiles.md
@@ -24,9 +24,9 @@ Of course, you wouldn’t want to type
     $ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn
 
 every time you want to run Subversion. Of course we could set up the
-PATH environment variable to include the `bin` directory of every
+`PATH` environment variable to include the `bin` directory of every
 package we want to use, but this is not very convenient since changing
-PATH doesn’t take effect for already existing processes. The solution
+`PATH` doesn’t take effect for already existing processes. The solution
 Nix uses is to create directory trees of symlinks to *activated*
 packages. These are called *user environments* and they are packages
 themselves (though automatically generated by `nix-env`), so they too
@@ -89,9 +89,9 @@ also see all available generations:
     $ nix-env --list-generations
 
 You generally wouldn’t have `/nix/var/nix/profiles/some-profile/bin` in
-your PATH. Rather, there is a symlink `~/.nix-profile` that points to
+your `PATH`. Rather, there is a symlink `~/.nix-profile` that points to
 your current profile. This means that you should put
-`~/.nix-profile/bin` in your PATH (and indeed, that’s what the
+`~/.nix-profile/bin` in your `PATH` (and indeed, that’s what the
 initialisation script `/nix/etc/profile.d/nix.sh` does). This makes it
 easier to switch to a different profile. You can do that using the
 command `nix-env --switch-profile`:
diff --git a/doc/manual/src/release-notes/rl-0.10.md b/doc/manual/src/release-notes/rl-0.10.md
index bf0e59fad..07b19ff05 100644
--- a/doc/manual/src/release-notes/rl-0.10.md
+++ b/doc/manual/src/release-notes/rl-0.10.md
@@ -124,8 +124,8 @@
             URL` allows a package to be installed directly from the given URL.
 
   - Nix now works behind an HTTP proxy server; just set the standard
-    environment variables http\_proxy, https\_proxy, ftp\_proxy or
-    all\_proxy appropriately. Functions such as `fetchurl` in Nixpkgs
+    environment variables `http_proxy`, `https_proxy`, `ftp_proxy` or
+    `all_proxy` appropriately. Functions such as `fetchurl` in Nixpkgs
     also respect these variables.
 
   - `nix-build -o
diff --git a/doc/manual/src/release-notes/rl-0.12.md b/doc/manual/src/release-notes/rl-0.12.md
index 0dc727b69..aaecfbb70 100644
--- a/doc/manual/src/release-notes/rl-0.12.md
+++ b/doc/manual/src/release-notes/rl-0.12.md
@@ -33,7 +33,7 @@
     (remote) Nix stores mounted somewhere in the filesystem. For
     instance, you can speed up an installation by mounting some remote
     Nix store that already has the packages in question via NFS or
-    `sshfs`. The environment variable NIX\_OTHER\_STORES specifies the
+    `sshfs`. The environment variable `NIX_OTHER_STORES` specifies the
     locations of the remote Nix directories, e.g. `/mnt/remote-fs/nix`.
 
   - New `nix-store` operations `--dump-db` and `--load-db` to dump and
@@ -111,7 +111,7 @@
         (integer multiplication), `builtins.div` (integer division).
 
   - `nix-prefetch-url` now supports `mirror://` URLs, provided that the
-    environment variable NIXPKGS\_ALL points at a Nixpkgs tree.
+    environment variable `NIXPKGS_ALL` points at a Nixpkgs tree.
 
   - Removed the commands `nix-pack-closure` and `nix-unpack-closure`.
     You can do almost the same thing but much more efficiently by doing
diff --git a/doc/manual/src/release-notes/rl-0.16.md b/doc/manual/src/release-notes/rl-0.16.md
index 121bbde28..79074fcdc 100644
--- a/doc/manual/src/release-notes/rl-0.16.md
+++ b/doc/manual/src/release-notes/rl-0.16.md
@@ -14,7 +14,7 @@ This release has the following improvements:
     performed in parallel was not configurable. Nix now has an option
     `--cores
                             N` as well as a configuration setting `build-cores =
-                            N` that causes the environment variable NIX\_BUILD\_CORES to be set
+                            N` that causes the environment variable `NIX_BUILD_CORES` to be set
     to N when the builder is invoked. The builder can use this at its
     discretion to perform a parallel build, e.g., by calling `make -j
                             N`. In Nixpkgs, this can be enabled on a per-package basis by
diff --git a/doc/manual/src/release-notes/rl-1.0.md b/doc/manual/src/release-notes/rl-1.0.md
index 025f827d9..cdb257787 100644
--- a/doc/manual/src/release-notes/rl-1.0.md
+++ b/doc/manual/src/release-notes/rl-1.0.md
@@ -14,7 +14,7 @@ release. Here are the most significant:
     significant speedup.
 
   - Nix now has an search path for expressions. The search path is set
-    using the environment variable NIX\_PATH and the `-I` command line
+    using the environment variable `NIX_PATH` and the `-I` command line
     option. In Nix expressions, paths between angle brackets are used to
     specify files that must be looked up in the search path. For
     instance, the expression `<nixpkgs/default.nix>` looks for a file
diff --git a/doc/manual/src/release-notes/rl-1.2.md b/doc/manual/src/release-notes/rl-1.2.md
index e62b2dac9..25b830955 100644
--- a/doc/manual/src/release-notes/rl-1.2.md
+++ b/doc/manual/src/release-notes/rl-1.2.md
@@ -77,7 +77,7 @@ This release has the following improvements and changes:
     about twice as fast.
 
   - Basic Nix expression evaluation profiling: setting the environment
-    variable NIX\_COUNT\_CALLS to `1` will cause Nix to print how many
+    variable `NIX_COUNT_CALLS` to `1` will cause Nix to print how many
     times each primop or function was executed.
 
   - New primops: `concatLists`, `elem`, `elemAt` and `filter`.
diff --git a/doc/manual/src/release-notes/rl-1.6.md b/doc/manual/src/release-notes/rl-1.6.md
index a4583f4e6..9b83d9274 100644
--- a/doc/manual/src/release-notes/rl-1.6.md
+++ b/doc/manual/src/release-notes/rl-1.6.md
@@ -13,7 +13,7 @@ features:
     get an environment that more closely corresponds to the “real” Nix
     build.
 
-  - `nix-shell` now sets the shell prompt (PS1) to ensure that Nix
+  - `nix-shell` now sets the shell prompt (`PS1`) to ensure that Nix
     shells are distinguishable from your regular shells.
 
   - `nix-env` no longer requires a `*` argument to match all packages,
diff --git a/doc/manual/src/release-notes/rl-1.8.md b/doc/manual/src/release-notes/rl-1.8.md
index 10a82d52a..59af363e8 100644
--- a/doc/manual/src/release-notes/rl-1.8.md
+++ b/doc/manual/src/release-notes/rl-1.8.md
@@ -76,7 +76,7 @@
     them anyway.
 
   - Various commands now automatically pipe their output into the pager
-    as specified by the PAGER environment variable.
+    as specified by the `PAGER` environment variable.
 
   - Several improvements to reduce memory consumption in the evaluator.
 
diff --git a/doc/manual/src/release-notes/rl-1.9.md b/doc/manual/src/release-notes/rl-1.9.md
index 01b067aab..92c6af90b 100644
--- a/doc/manual/src/release-notes/rl-1.9.md
+++ b/doc/manual/src/release-notes/rl-1.9.md
@@ -39,8 +39,8 @@ features:
         This builds GNU Hello from the latest revision of the Nixpkgs
         master branch.
     
-      - In the Nix search path (as specified via NIX\_PATH or `-I`). For
-        example, to start a shell containing the Pan package from a
+      - In the Nix search path (as specified via `NIX_PATH` or `-I`).
+        For example, to start a shell containing the Pan package from a
         specific version of Nixpkgs:
         
             $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
@@ -124,8 +124,8 @@ features:
   - `nix-env` now only creates a new “generation” symlink in
     `/nix/var/nix/profiles` if something actually changed.
 
-  - The environment variable NIX\_PAGER can now be set to override
-    PAGER. You can set it to `cat` to disable paging for Nix commands
+  - The environment variable `NIX_PAGER` can now be set to override
+    `PAGER`. You can set it to `cat` to disable paging for Nix commands
     only.
 
   - Failing `<...>` lookups now show position information.
diff --git a/doc/manual/src/release-notes/rl-2.0.md b/doc/manual/src/release-notes/rl-2.0.md
index 0ce985b2f..170275f89 100644
--- a/doc/manual/src/release-notes/rl-2.0.md
+++ b/doc/manual/src/release-notes/rl-2.0.md
@@ -18,7 +18,7 @@ The following incompatible changes have been made:
       - `bspatch`
 
   - The “copy from other stores” substituter mechanism
-    (`copy-from-other-stores` and the NIX\_OTHER\_STORES environment
+    (`copy-from-other-stores` and the `NIX_OTHER_STORES` environment
     variable) has been removed. It was primarily used by the NixOS
     installer to copy available paths from the installation medium. The
     replacement is to use a chroot store as a substituter (e.g.
@@ -233,7 +233,7 @@ This release has the following new features:
         store, the latter via the Nix daemon. You can use `auto` or the
         empty string to auto-select a local or daemon store depending on
         whether you have write permission to the Nix store. It is no
-        longer necessary to set the NIX\_REMOTE environment variable to
+        longer necessary to set the `NIX_REMOTE` environment variable to
         use the Nix daemon.
         
         As noted above, `LocalStore` now supports chroot builds,
@@ -296,7 +296,7 @@ This release has the following new features:
         [now](https://github.com/NixOS/nix/commit/eba840c8a13b465ace90172ff76a0db2899ab11b)
         use `/build` instead of `/tmp` as the temporary build directory.
         This fixes potential security problems when a build accidentally
-        stores its TMPDIR in some security-sensitive place, such as an
+        stores its `TMPDIR` in some security-sensitive place, such as an
         RPATH.
 
   - *Pure evaluation mode*. With the `--pure-eval` flag, Nix enables a
@@ -334,8 +334,8 @@ This release has the following new features:
     using the Nix daemon, you can now just specify a remote build
     machine on the command line, e.g. `--option builders
                     'ssh://my-mac x86_64-darwin'`. The environment variable
-    NIX\_BUILD\_HOOK has been removed and is no longer needed. The
-    environment variable NIX\_REMOTE\_SYSTEMS is still supported for
+    `NIX_BUILD_HOOK` has been removed and is no longer needed. The
+    environment variable `NIX_REMOTE_SYSTEMS` is still supported for
     compatibility, but it is also possible to specify builders in
     `nix.conf` by setting the option `builders =
                     @path`.
@@ -353,7 +353,7 @@ This release has the following new features:
     Nixpkgs provides `lib.inNixShell` to check this variable during
     evaluation.
 
-  - NIX\_PATH is now lazy, so URIs in the path are only downloaded if
+  - `NIX_PATH` is now lazy, so URIs in the path are only downloaded if
     they are needed for evaluation.
 
   - You can now use <channel:> as a short-hand for
@@ -406,7 +406,7 @@ This release has the following new features:
     non-standard base-32.
 
   - `nix-shell` now uses `bashInteractive` from Nixpkgs, rather than the
-    `bash` command that happens to be in the caller’s PATH. This is
+    `bash` command that happens to be in the caller’s `PATH`. This is
     especially important on macOS where the `bash` provided by the
     system is seriously outdated and cannot execute `stdenv`’s setup
     script.
@@ -433,7 +433,7 @@ The Nix language has the following new features:
     
         configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";
     
-    will cause the configureFlags environment variable to contain the
+    will cause the `configureFlags` environment variable to contain the
     actual store paths corresponding to the `out` and `dev` outputs.
 
 The following builtin functions are new or extended:
@@ -481,23 +481,23 @@ The Nix build environment has the following changes:
     be passed to builders in a non-lossy way. If the special attribute
     `__structuredAttrs` is set to `true`, the other derivation
     attributes are serialised in JSON format and made available to the
-    builder via the file .attrs.json in the builder’s temporary
+    builder via the file `.attrs.json` in the builder’s temporary
     directory. This obviates the need for `passAsFile` since JSON files
     have no size restrictions, unlike process environments.
     
     [As a convenience to Bash
     builders](https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699),
-    Nix writes a script named .attrs.sh to the builder’s directory that
-    initialises shell variables corresponding to all attributes that are
-    representable in Bash. This includes non-nested (associative)
-    arrays. For example, the attribute `hardening.format =
+    Nix writes a script named `.attrs.sh` to the builder’s directory
+    that initialises shell variables corresponding to all attributes
+    that are representable in Bash. This includes non-nested
+    (associative) arrays. For example, the attribute `hardening.format =
                     true` ends up as the Bash associative array element
     `${hardening[format]}`.
 
   - Builders can
     [now](https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3)
     communicate what build phase they are in by writing messages to the
-    file descriptor specified in NIX\_LOG\_FD. The current phase is
+    file descriptor specified in `NIX_LOG_FD`. The current phase is
     shown by the `nix` progress indicator.
 
   - In Linux sandbox builds, we
diff --git a/doc/manual/src/release-notes/rl-2.2.md b/doc/manual/src/release-notes/rl-2.2.md
index 3667a48dc..b67d65db7 100644
--- a/doc/manual/src/release-notes/rl-2.2.md
+++ b/doc/manual/src/release-notes/rl-2.2.md
@@ -69,7 +69,7 @@ This is primarily a bug fix release. It also has the following changes:
   - Integers are now 64 bits on all platforms.
 
   - The evaluator now prints profiling statistics (enabled via the
-    NIX\_SHOW\_STATS and NIX\_COUNT\_CALLS environment variables) in
+    `NIX_SHOW_STATS` and `NIX_COUNT_CALLS` environment variables) in
     JSON format.
 
   - The option `--xml` in `nix-store
diff --git a/doc/manual/src/release-notes/rl-2.3.md b/doc/manual/src/release-notes/rl-2.3.md
index a4bb6ebb0..d1f4e3734 100644
--- a/doc/manual/src/release-notes/rl-2.3.md
+++ b/doc/manual/src/release-notes/rl-2.3.md
@@ -25,7 +25,7 @@ It also has the following changes:
     line in the progress bar. To distinguish between concurrent builds,
     log lines are prefixed by the name of the package.
 
-  - Builds are now executed in a pseudo-terminal, and the TERM
+  - Builds are now executed in a pseudo-terminal, and the `TERM`
     environment variable is set to `xterm-256color`. This allows many
     programs (e.g. `gcc`, `clang`, `cmake`) to print colorized log
     output.

From efdb89994c2ee79763320d70fb4ba881d0e3b1e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 10:56:15 +0200
Subject: [PATCH 027/882] Convert nix.conf manpage

---
 doc/manual/command-ref/conf-file.xml    |  43 +-
 doc/manual/local.mk                     |   6 +-
 doc/manual/src/SUMMARY.md               |   2 +
 doc/manual/src/command-ref/conf-file.md | 727 ++++++++++++++++++++++++
 doc/manual/src/command-ref/files.md     |   4 +
 5 files changed, 763 insertions(+), 19 deletions(-)
 create mode 100644 doc/manual/src/command-ref/conf-file.md
 create mode 100644 doc/manual/src/command-ref/files.md

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index ff9c7d46c..4c103f505 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -21,28 +21,35 @@
 
 <para>By default Nix reads settings from the following places:</para>
 
-<para>The system-wide configuration file
-<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-<filename>$NIX_CONF_DIR/nix.conf</filename> if
-<literal>NIX_CONF_DIR</literal> is set. Values loaded in this file are not forwarded to the Nix daemon. The
-client assumes that the daemon has already loaded them.
-</para>
+<itemizedlist>
 
-<para>User-specific configuration files:</para>
+  <listitem>
+    <para>
+      The system-wide configuration file
+      <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+      (i.e. <filename>/etc/nix/nix.conf</filename> on most systems),
+      or <filename>$NIX_CONF_DIR/nix.conf</filename> if
+      <literal>NIX_CONF_DIR</literal> is set. Values loaded in this
+      file are not forwarded to the Nix daemon. The client assumes
+      that the daemon has already loaded them.
+    </para>
+  </listitem>
 
-<para>
-  If <literal>NIX_USER_CONF_FILES</literal> is set, then each path separated by
-  <literal>:</literal> will be loaded in reverse order.
-</para>
+  <listitem>
+    <para>
+      If <literal>NIX_USER_CONF_FILES</literal> is set, then each path separated by
+      <literal>:</literal> will be loaded in reverse order.
+    </para>
 
-<para>
-  Otherwise it will look for <filename>nix/nix.conf</filename> files in
-  <literal>XDG_CONFIG_DIRS</literal> and <literal>XDG_CONFIG_HOME</literal>.
+    <para>
+      Otherwise it will look for <filename>nix/nix.conf</filename>
+      files in <literal>XDG_CONFIG_DIRS</literal> and
+      <literal>XDG_CONFIG_HOME</literal>.  If these are unset, it will
+      look in <filename>$HOME/.config/nix.conf</filename>.
+    </para>
+  </listitem>
 
-  The default location is <filename>$HOME/.config/nix.conf</filename> if
-  those environment variables are unset.
-</para>
+</itemizedlist>
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 65acd658c..80c517702 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -12,7 +12,8 @@ dist-files += $(d)/version.txt
 
 # Generate man pages.
 man-pages := $(foreach n, \
-  nix-copy-closure.1, \
+  nix-copy-closure.1 \
+  nix.conf.5, \
   $(d)/$(n))
 #  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
 #  nix-collect-garbage.1, \
@@ -27,6 +28,9 @@ dist-files += $(man-pages)
 $(d)/nix-copy-closure.1: $(d)/src/command-ref/nix-copy-closure.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
+$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
+	$(trace-gen) lowdown -sT man $^ -o $@
+
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index f9ef1b060..e8baf489b 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -49,6 +49,8 @@
   - [Common Environment Variables](command-ref/env-common.md)
   - [Utilities](command-ref/utilities.md)
     - [nix-copy-closure](command-ref/nix-copy-closure.md)
+  - [Files](command-ref/files.md)
+    - [nix.conf](command-ref/conf-file.md)
 - [Glossary](glossary.md)
 - [Hacking](hacking.md)
 - [Release Notes](release-notes/release-notes.md)
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
new file mode 100644
index 000000000..3f84373e2
--- /dev/null
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -0,0 +1,727 @@
+nix.conf
+
+5
+
+Nix
+
+nix.conf
+
+Nix configuration file
+
+# Description
+
+By default Nix reads settings from the following places:
+
+  - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e.
+    `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if
+    `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded
+    to the Nix daemon. The client assumes that the daemon has already
+    loaded them.
+
+  - If `NIX_USER_CONF_FILES` is set, then each path separated by `:`
+    will be loaded in reverse order.
+    
+    Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
+    and `XDG_CONFIG_HOME`. If these are unset, it will look in
+    `$HOME/.config/nix.conf`.
+
+The configuration files consist of `name =
+value` pairs, one per line. Other files can be included with a line like
+`include
+path`, where path is interpreted relative to the current conf file and a
+missing file is an error unless `!include` is used instead. Comments
+start with a `#` character. Here is an example configuration file:
+
+    keep-outputs = true       # Nice for developers
+    keep-derivations = true   # Idem
+
+You can override settings on the command line using the `--option` flag,
+e.g. `--option keep-outputs
+false`.
+
+The following settings are currently available:
+
+  - `allowed-uris`  
+    A list of URI prefixes to which access is allowed in restricted
+    evaluation mode. For example, when set to
+    `https://github.com/NixOS`, builtin functions such as `fetchGit` are
+    allowed to access `https://github.com/NixOS/patchelf.git`.
+
+  - `allow-import-from-derivation`  
+    By default, Nix allows you to `import` from a derivation, allowing
+    building at evaluation time. With this option set to false, Nix will
+    throw an error when evaluating an expression that uses this feature,
+    allowing users to ensure their evaluation will not require any
+    builds to take place.
+
+  - `allow-new-privileges`  
+    (Linux-specific.) By default, builders on Linux cannot acquire new
+    privileges by calling setuid/setgid programs or programs that have
+    file capabilities. For example, programs such as `sudo` or `ping`
+    will fail. (Note that in sandbox builds, no such programs are
+    available unless you bind-mount them into the sandbox via the
+    `sandbox-paths` option.) You can allow the use of such programs by
+    enabling this option. This is impure and usually undesirable, but
+    may be useful in certain scenarios (e.g. to spin up containers or
+    set up userspace network interfaces in tests).
+
+  - `allowed-users`  
+    A list of names of users (separated by whitespace) that are allowed
+    to connect to the Nix daemon. As with the `trusted-users` option,
+    you can specify groups by prefixing them with `@`. Also, you can
+    allow all users by specifying `*`. The default is `*`.
+    
+    Note that trusted users are always allowed to connect.
+
+  - `auto-optimise-store`  
+    If set to `true`, Nix automatically detects files in the store that
+    have identical contents, and replaces them with hard links to a
+    single copy. This saves disk space. If set to `false` (the default),
+    you can still run `nix-store
+                    --optimise` to get rid of duplicate files.
+
+  - `builders`  
+    A list of machines on which to perform builds. See
+    [???](#chap-distributed-builds) for details.
+
+  - `builders-use-substitutes`  
+    If set to `true`, Nix will instruct remote build machines to use
+    their own binary substitutes if available. In practical terms, this
+    means that remote hosts will fetch as many build dependencies as
+    possible from their own substitutes (e.g, from `cache.nixos.org`),
+    instead of waiting for this host to upload them all. This can
+    drastically reduce build times if the network connection between
+    this computer and the remote build host is slow. Defaults to
+    `false`.
+
+  - `build-users-group`  
+    This options specifies the Unix group containing the Nix build user
+    accounts. In multi-user Nix installations, builds should not be
+    performed by the Nix account since that would allow users to
+    arbitrarily modify the Nix store and database by supplying specially
+    crafted builders; and they cannot be performed by the calling user
+    since that would allow him/her to influence the build result.
+    
+    Therefore, if this option is non-empty and specifies a valid group,
+    builds will be performed under the user accounts that are a member
+    of the group specified here (as listed in `/etc/group`). Those user
+    accounts should not be used for any other purpose\!
+    
+    Nix will never run two builds under the same user account at the
+    same time. This is to prevent an obvious security hole: a malicious
+    user writing a Nix expression that modifies the build result of a
+    legitimate Nix expression being built by another user. Therefore it
+    is good to have as many Nix build user accounts as you can spare.
+    (Remember: uids are cheap.)
+    
+    The build users should have permission to create files in the Nix
+    store, but not delete them. Therefore, `/nix/store` should be owned
+    by the Nix account, its group should be the group specified here,
+    and its mode should be `1775`.
+    
+    If the build users group is empty, builds will be performed under
+    the uid of the Nix process (that is, the uid of the caller if
+    `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
+    `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
+    multi-user settings with untrusted users.
+
+  - `compress-build-log`  
+    If set to `true` (the default), build logs written to
+    `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2.
+    Otherwise, they will not be compressed.
+
+  - `connect-timeout`  
+    The timeout (in seconds) for establishing connections in the binary
+    cache substituter. It corresponds to `curl`’s `--connect-timeout`
+    option.
+
+  - `cores`  
+    Sets the value of the `NIX_BUILD_CORES` environment variable in the
+    invocation of builders. Builders can use this variable at their
+    discretion to control the maximum amount of parallelism. For
+    instance, in Nixpkgs, if the derivation attribute
+    `enableParallelBuilding` is set to `true`, the builder passes the
+    `-jN` flag to GNU Make. It can be overridden using the `--cores`
+    command line switch and defaults to `1`. The value `0` means that
+    the builder should use all available CPU cores in the system.
+    
+    See also [???](#chap-tuning-cores-and-jobs).
+
+  - `diff-hook`  
+    Absolute path to an executable capable of diffing build results. The
+    hook executes if [varlistentry\_title](#conf-run-diff-hook) is true,
+    and the output of a build is known to not be the same. This program
+    is not executed to determine if two results are the same.
+    
+    The diff hook is executed by the same user and group who ran the
+    build. However, the diff hook does not have write access to the
+    store path just built.
+    
+    The diff hook program receives three parameters:
+    
+    1.  A path to the previous build's results
+    
+    2.  A path to the current build's results
+    
+    3.  The path to the build's derivation
+    
+    4.  The path to the build's scratch directory. This directory will
+        exist only if the build was run with `--keep-failed`.
+    
+    The stderr and stdout output from the diff hook will not be
+    displayed to the user. Instead, it will print to the nix-daemon's
+    log.
+    
+    When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
+    configuration file, and cannot be passed at the command line.
+
+  - `enforce-determinism`  
+    See [varlistentry\_title](#conf-repeat).
+
+  - `extra-sandbox-paths`  
+    A list of additional paths appended to `sandbox-paths`. Useful if
+    you want to extend its default value.
+
+  - `extra-platforms`  
+    Platforms other than the native one which this machine is capable of
+    building for. This can be useful for supporting additional
+    architectures on compatible machines: i686-linux can be built on
+    x86\_64-linux machines (and the default for this setting reflects
+    this); armv7 is backwards-compatible with armv6 and armv5tel; some
+    aarch64 machines can also natively run 32-bit ARM code; and
+    qemu-user may be used to support non-native platforms (though this
+    may be slow and buggy). Most values for this are not enabled by
+    default because build systems will often misdetect the target
+    platform and generate incompatible code, so you may wish to
+    cross-check the results of using this option against proper
+    natively-built versions of your derivations.
+
+  - `extra-substituters`  
+    Additional binary caches appended to those specified in
+    `substituters`. When used by unprivileged users, untrusted
+    substituters (i.e. those not listed in `trusted-substituters`) are
+    silently ignored.
+
+  - `fallback`  
+    If set to `true`, Nix will fall back to building from source if a
+    binary substitute fails. This is equivalent to the `--fallback`
+    flag. The default is `false`.
+
+  - `fsync-metadata`  
+    If set to `true`, changes to the Nix store metadata (in
+    `/nix/var/nix/db`) are synchronously flushed to disk. This improves
+    robustness in case of system crashes, but reduces performance. The
+    default is `true`.
+
+  - `hashed-mirrors`  
+    A list of web servers used by `builtins.fetchurl` to obtain files by
+    hash. The default is `http://tarballs.nixos.org/`. Given a hash type
+    ht and a base-16 hash h, Nix will try to download the file from
+    `hashed-mirror/ht/h`. This allows files to be downloaded even if
+    they have disappeared from their original URI. For example, given
+    the default mirror `http://tarballs.nixos.org/`, when building the
+    derivation
+    
+        builtins.fetchurl {
+          url = "https://example.org/foo-1.2.3.tar.xz";
+          sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+        }
+    
+    Nix will attempt to download this file from
+    `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
+    first. If it is not available there, if will try the original URI.
+
+  - `http-connections`  
+    The maximum number of parallel TCP connections used to fetch files
+    from binary caches and by other downloads. It defaults to 25. 0
+    means no limit.
+
+  - `keep-build-log`  
+    If set to `true` (the default), Nix will write the build log of a
+    derivation (i.e. the standard output and error of its builder) to
+    the directory `/nix/var/log/nix/drvs`. The build log can be
+    retrieved using the command `nix-store -l
+                    path`.
+
+  - `keep-derivations`  
+    If `true` (default), the garbage collector will keep the derivations
+    from which non-garbage store paths were built. If `false`, they will
+    be deleted unless explicitly registered as a root (or reachable from
+    other roots).
+    
+    Keeping derivation around is useful for querying and traceability
+    (e.g., it allows you to ask with what dependencies or options a
+    store path was built), so by default this option is on. Turn it off
+    to save a bit of disk space (or a lot if `keep-outputs` is also
+    turned on).
+
+  - `keep-env-derivations`  
+    If `false` (default), derivations are not stored in Nix user
+    environments. That is, the derivations of any build-time-only
+    dependencies may be garbage-collected.
+    
+    If `true`, when you add a Nix derivation to a user environment, the
+    path of the derivation is stored in the user environment. Thus, the
+    derivation will not be garbage-collected until the user environment
+    generation is deleted (`nix-env --delete-generations`). To prevent
+    build-time-only dependencies from being collected, you should also
+    turn on `keep-outputs`.
+    
+    The difference between this option and `keep-derivations` is that
+    this one is “sticky”: it applies to any user environment created
+    while this option was enabled, while `keep-derivations` only applies
+    at the moment the garbage collector is run.
+
+  - `keep-outputs`  
+    If `true`, the garbage collector will keep the outputs of
+    non-garbage derivations. If `false` (default), outputs will be
+    deleted unless they are GC roots themselves (or reachable from other
+    roots).
+    
+    In general, outputs must be registered as roots separately. However,
+    even if the output of a derivation is registered as a root, the
+    collector will still delete store paths that are used only at build
+    time (e.g., the C compiler, or source tarballs downloaded from the
+    network). To prevent it from doing so, set this option to `true`.
+
+  - `max-build-log-size`  
+    This option defines the maximum number of bytes that a builder can
+    write to its stdout/stderr. If the builder exceeds this limit, it’s
+    killed. A value of `0` (the default) means that there is no limit.
+
+  - `max-free`  
+    When a garbage collection is triggered by the `min-free` option, it
+    stops as soon as `max-free` bytes are available. The default is
+    infinity (i.e. delete all garbage).
+
+  - `max-jobs`  
+    This option defines the maximum number of jobs that Nix will try to
+    build in parallel. The default is `1`. The special value `auto`
+    causes Nix to use the number of CPUs in your system. `0` is useful
+    when using remote builders to prevent any local builds (except for
+    `preferLocalBuild` derivation attribute which executes locally
+    regardless). It can be overridden using the `--max-jobs` (`-j`)
+    command line switch.
+    
+    See also [???](#chap-tuning-cores-and-jobs).
+
+  - `max-silent-time`  
+    This option defines the maximum number of seconds that a builder can
+    go without producing any data on standard output or standard error.
+    This is useful (for instance in an automated build system) to catch
+    builds that are stuck in an infinite loop, or to catch remote builds
+    that are hanging due to network problems. It can be overridden using
+    the `--max-silent-time` command line switch.
+    
+    The value `0` means that there is no timeout. This is also the
+    default.
+
+  - `min-free`  
+    When free disk space in `/nix/store` drops below `min-free` during a
+    build, Nix performs a garbage-collection until `max-free` bytes are
+    available or there is no more garbage. A value of `0` (the default)
+    disables this feature.
+
+  - `narinfo-cache-negative-ttl`  
+    The TTL in seconds for negative lookups. If a store path is queried
+    from a substituter but was not found, there will be a negative
+    lookup cached in the local disk cache database for the specified
+    duration.
+
+  - `narinfo-cache-positive-ttl`  
+    The TTL in seconds for positive lookups. If a store path is queried
+    from a substituter, the result of the query will be cached in the
+    local disk cache database including some of the NAR metadata. The
+    default TTL is a month, setting a shorter TTL for positive lookups
+    can be useful for binary caches that have frequent garbage
+    collection, in which case having a more frequent cache invalidation
+    would prevent trying to pull the path again and failing with a hash
+    mismatch if the build isn't reproducible.
+
+  - `netrc-file`  
+    If set to an absolute path to a `netrc` file, Nix will use the HTTP
+    authentication credentials in this file when trying to download from
+    a remote host through HTTP or HTTPS. Defaults to
+    `$NIX_CONF_DIR/netrc`.
+    
+    The `netrc` file consists of a list of accounts in the following
+    format:
+    
+        machine my-machine
+        login my-username
+        password my-password
+    
+    For the exact syntax, see [the `curl`
+    documentation.](https://ec.haxx.se/usingcurl-netrc.html)
+    
+    > **Note**
+    > 
+    > This must be an absolute path, and `~` is not resolved. For
+    > example, `~/.netrc` won't resolve to your home directory's
+    > `.netrc`.
+
+  - `plugin-files`  
+    A list of plugin files to be loaded by Nix. Each of these files will
+    be dlopened by Nix, allowing them to affect execution through static
+    initialization. In particular, these plugins may construct static
+    instances of RegisterPrimOp to add new primops or constants to the
+    expression language, RegisterStoreImplementation to add new store
+    implementations, RegisterCommand to add new subcommands to the `nix`
+    command, and RegisterSetting to add new nix config settings. See the
+    constructors for those types for more details.
+    
+    Since these files are loaded into the same address space as Nix
+    itself, they must be DSOs compatible with the instance of Nix
+    running at the time (i.e. compiled against the same headers, not
+    linked to any incompatible libraries). They should not be linked to
+    any Nix libs directly, as those will be available already at load
+    time.
+    
+    If an entry in the list is a directory, all files in the directory
+    are loaded as plugins (non-recursively).
+
+  - `pre-build-hook`  
+    If set, the path to a program that can set extra derivation-specific
+    settings for this system. This is used for settings that can't be
+    captured by the derivation model itself and are too variable between
+    different versions of the same system to be hard-coded into nix.
+    
+    The hook is passed the derivation path and, if sandboxes are
+    enabled, the sandbox directory. It can then modify the sandbox and
+    send a series of commands to modify various settings to stdout. The
+    currently recognized commands are:
+    
+      - `extra-sandbox-paths`  
+        Pass a list of files and directories to be included in the
+        sandbox for this build. One entry per line, terminated by an
+        empty line. Entries have the same format as `sandbox-paths`.
+
+  - `post-build-hook`  
+    Optional. The path to a program to execute after each build.
+    
+    This option is only settable in the global `nix.conf`, or on the
+    command line by trusted users.
+    
+    When using the nix-daemon, the daemon executes the hook as `root`.
+    If the nix-daemon is not involved, the hook runs as the user
+    executing the nix-build.
+    
+      - The hook executes after an evaluation-time build.
+    
+      - The hook does not execute on substituted paths.
+    
+      - The hook's output always goes to the user's terminal.
+    
+      - If the hook fails, the build succeeds but no further builds
+        execute.
+    
+      - The hook executes synchronously, and blocks other builds from
+        progressing while it runs.
+    
+    The program executes with no arguments. The program's environment
+    contains the following environment variables:
+    
+      - `DRV_PATH`  
+        The derivation for the built paths.
+        
+        Example:
+        `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
+    
+      - `OUT_PATHS`  
+        Output paths of the built derivation, separated by a space
+        character.
+        
+        Example:
+        `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+                                                                                                        /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+                                                                                                        /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+                                                                                                        /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+                                                                                                        /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
+    
+    See [???](#chap-post-build-hook) for an example implementation.
+
+  - `repeat`  
+    How many times to repeat builds to check whether they are
+    deterministic. The default value is 0. If the value is non-zero,
+    every build is repeated the specified number of times. If the
+    contents of any of the runs differs from the previous ones and
+    [varlistentry\_title](#conf-enforce-determinism) is true, the build
+    is rejected and the resulting store paths are not registered as
+    “valid” in Nix’s database.
+
+  - `require-sigs`  
+    If set to `true` (the default), any non-content-addressed path added
+    or copied to the Nix store (e.g. when substituting from a binary
+    cache) must have a valid signature, that is, be signed using one of
+    the keys listed in `trusted-public-keys` or `secret-key-files`. Set
+    to `false` to disable signature checking.
+
+  - `restrict-eval`  
+    If set to `true`, the Nix evaluator will not allow access to any
+    files outside of the Nix search path (as set via the `NIX_PATH`
+    environment variable or the `-I` option), or to URIs outside of
+    `allowed-uri`. The default is `false`.
+
+  - `run-diff-hook`  
+    If true, enable the execution of
+    [varlistentry\_title](#conf-diff-hook).
+    
+    When using the Nix daemon, `run-diff-hook` must be set in the
+    `nix.conf` configuration file, and cannot be passed at the command
+    line.
+
+  - `sandbox`  
+    If set to `true`, builds will be performed in a *sandboxed
+    environment*, i.e., they’re isolated from the normal file system
+    hierarchy and will only see their dependencies in the Nix store, the
+    temporary build directory, private versions of `/proc`, `/dev`,
+    `/dev/shm` and `/dev/pts` (on Linux), and the paths configured with
+    the [`sandbox-paths` option](#conf-sandbox-paths). This is useful to
+    prevent undeclared dependencies on files in directories such as
+    `/usr/bin`. In addition, on Linux, builds run in private PID, mount,
+    network, IPC and UTS namespaces to isolate them from other processes
+    in the system (except that fixed-output derivations do not run in
+    private network namespace to ensure they can access the network).
+    
+    Currently, sandboxing only work on Linux and macOS. The use of a
+    sandbox requires that Nix is run as root (so you should use the
+    [“build users” feature](#conf-build-users-group) to perform the
+    actual builds under different users than root).
+    
+    If this option is set to `relaxed`, then fixed-output derivations
+    and derivations that have the `__noChroot` attribute set to `true`
+    do not run in sandboxes.
+    
+    The default is `true` on Linux and `false` on all other platforms.
+
+  - `sandbox-dev-shm-size`  
+    This option determines the maximum size of the `tmpfs` filesystem
+    mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+    description of the `size` option of `tmpfs` in mount8. The default
+    is `50%`.
+
+  - `sandbox-paths`  
+    A list of paths bind-mounted into Nix sandbox environments. You can
+    use the syntax `target=source` to mount a path in a different
+    location in the sandbox; for instance, `/bin=/nix-bin` will mount
+    the path `/nix-bin` as `/bin` inside the sandbox. If source is
+    followed by `?`, then it is not an error if source does not exist;
+    for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
+    only be mounted in the sandbox if it exists in the host filesystem.
+    
+    Depending on how Nix was built, the default value for this option
+    may be empty or provide `/bin/sh` as a bind-mount of `bash`.
+
+  - `secret-key-files`  
+    A whitespace-separated list of files containing secret (private)
+    keys. These are used to sign locally-built paths. They can be
+    generated using `nix-store
+                    --generate-binary-cache-key`. The corresponding public key can be
+    distributed to other users, who can add it to `trusted-public-keys`
+    in their `nix.conf`.
+
+  - `show-trace`  
+    Causes Nix to print out a stack trace in case of Nix expression
+    evaluation errors.
+
+  - `substitute`  
+    If set to `true` (default), Nix will use binary substitutes if
+    available. This option can be disabled to force building from
+    source.
+
+  - `stalled-download-timeout`  
+    The timeout (in seconds) for receiving data from servers during
+    download. Nix cancels idle downloads after this timeout's duration.
+
+  - `substituters`  
+    A list of URLs of substituters, separated by whitespace. The default
+    is `https://cache.nixos.org`.
+
+  - `system`  
+    This option specifies the canonical Nix system name of the current
+    installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
+    build derivations whose `system` attribute equals the value
+    specified here. In general, it never makes sense to modify this
+    value from its default, since you can use it to ‘lie’ about the
+    platform you are building on (e.g., perform a Mac OS build on a
+    Linux machine; the result would obviously be wrong). It only makes
+    sense if the Nix binaries can run on multiple platforms, e.g.,
+    ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
+    
+    It defaults to the canonical Nix system name detected by `configure`
+    at build time.
+
+  - `system-features`  
+    A set of system “features” supported by this machine, e.g. `kvm`.
+    Derivations can express a dependency on such features through the
+    derivation attribute `requiredSystemFeatures`. For example, the
+    attribute
+    
+        requiredSystemFeatures = [ "kvm" ];
+    
+    ensures that the derivation can only be built on a machine with the
+    `kvm` feature.
+    
+    This setting by default includes `kvm` if `/dev/kvm` is accessible,
+    and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
+    that are used in Nixpkgs to route builds to specific machines.
+
+  - `tarball-ttl`  
+    Default: `3600` seconds.
+    
+    The number of seconds a downloaded tarball is considered fresh. If
+    the cached tarball is stale, Nix will check whether it is still up
+    to date using the ETag header. Nix will download a new version if
+    the ETag header is unsupported, or the cached ETag doesn't match.
+    
+    Setting the TTL to `0` forces Nix to always check if the tarball is
+    up to date.
+    
+    Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
+    
+    Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
+    `fetchTarball`, and `fetchurl` respect this TTL.
+
+  - `timeout`  
+    This option defines the maximum number of seconds that a builder can
+    run. This is useful (for instance in an automated build system) to
+    catch builds that are stuck in an infinite loop but keep writing to
+    their standard output or standard error. It can be overridden using
+    the `--timeout` command line switch.
+    
+    The value `0` means that there is no timeout. This is also the
+    default.
+
+  - `trace-function-calls`  
+    Default: `false`.
+    
+    If set to `true`, the Nix evaluator will trace every function call.
+    Nix will print a log message at the "vomit" level for every function
+    entrance and function exit.
+    
+    <div class="informalexample">
+    
+        function-trace entered undefined position at 1565795816999559622
+        function-trace exited undefined position at 1565795816999581277
+        function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+        function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+    
+    </div>
+    
+    The `undefined position` means the function call is a builtin.
+    
+    Use the `contrib/stack-collapse.py` script distributed with the Nix
+    source code to convert the trace logs in to a format suitable for
+    `flamegraph.pl`.
+
+  - `trusted-public-keys`  
+    A whitespace-separated list of public keys. When paths are copied
+    from another Nix store (such as a binary cache), they must be signed
+    with one of these keys. For example:
+    `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+                    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
+
+  - `trusted-substituters`  
+    A list of URLs of substituters, separated by whitespace. These are
+    not used by default, but can be enabled by users of the Nix daemon
+    by specifying `--option
+                    substituters urls` on the command line. Unprivileged users are only
+    allowed to pass a subset of the URLs listed in `substituters` and
+    `trusted-substituters`.
+
+  - `trusted-users`  
+    A list of names of users (separated by whitespace) that have
+    additional rights when connecting to the Nix daemon, such as the
+    ability to specify additional binary caches, or to import unsigned
+    NARs. You can also specify groups by prefixing them with `@`; for
+    instance, `@wheel` means all users in the `wheel` group. The default
+    is `root`.
+    
+    > **Warning**
+    > 
+    > Adding a user to `trusted-users` is essentially equivalent to
+    > giving that user root access to the system. For example, the user
+    > can set `sandbox-paths` and thereby obtain read access to
+    > directories that are otherwise inacessible to them.
+
+## Deprecated Settings
+
+  - `binary-caches`  
+    *Deprecated:* `binary-caches` is now an alias to
+    [varlistentry\_title](#conf-substituters).
+
+  - `binary-cache-public-keys`  
+    *Deprecated:* `binary-cache-public-keys` is now an alias to
+    [varlistentry\_title](#conf-trusted-public-keys).
+
+  - `build-compress-log`  
+    *Deprecated:* `build-compress-log` is now an alias to
+    [varlistentry\_title](#conf-compress-build-log).
+
+  - `build-cores`  
+    *Deprecated:* `build-cores` is now an alias to
+    [varlistentry\_title](#conf-cores).
+
+  - `build-extra-chroot-dirs`  
+    *Deprecated:* `build-extra-chroot-dirs` is now an alias to
+    [varlistentry\_title](#conf-extra-sandbox-paths).
+
+  - `build-extra-sandbox-paths`  
+    *Deprecated:* `build-extra-sandbox-paths` is now an alias to
+    [varlistentry\_title](#conf-extra-sandbox-paths).
+
+  - `build-fallback`  
+    *Deprecated:* `build-fallback` is now an alias to
+    [varlistentry\_title](#conf-fallback).
+
+  - `build-max-jobs`  
+    *Deprecated:* `build-max-jobs` is now an alias to
+    [varlistentry\_title](#conf-max-jobs).
+
+  - `build-max-log-size`  
+    *Deprecated:* `build-max-log-size` is now an alias to
+    [varlistentry\_title](#conf-max-build-log-size).
+
+  - `build-max-silent-time`  
+    *Deprecated:* `build-max-silent-time` is now an alias to
+    [varlistentry\_title](#conf-max-silent-time).
+
+  - `build-repeat`  
+    *Deprecated:* `build-repeat` is now an alias to
+    [varlistentry\_title](#conf-repeat).
+
+  - `build-timeout`  
+    *Deprecated:* `build-timeout` is now an alias to
+    [varlistentry\_title](#conf-timeout).
+
+  - `build-use-chroot`  
+    *Deprecated:* `build-use-chroot` is now an alias to
+    [varlistentry\_title](#conf-sandbox).
+
+  - `build-use-sandbox`  
+    *Deprecated:* `build-use-sandbox` is now an alias to
+    [varlistentry\_title](#conf-sandbox).
+
+  - `build-use-substitutes`  
+    *Deprecated:* `build-use-substitutes` is now an alias to
+    [varlistentry\_title](#conf-substitute).
+
+  - `gc-keep-derivations`  
+    *Deprecated:* `gc-keep-derivations` is now an alias to
+    [varlistentry\_title](#conf-keep-derivations).
+
+  - `gc-keep-outputs`  
+    *Deprecated:* `gc-keep-outputs` is now an alias to
+    [varlistentry\_title](#conf-keep-outputs).
+
+  - `env-keep-derivations`  
+    *Deprecated:* `env-keep-derivations` is now an alias to
+    [varlistentry\_title](#conf-keep-env-derivations).
+
+  - `extra-binary-caches`  
+    *Deprecated:* `extra-binary-caches` is now an alias to
+    [varlistentry\_title](#conf-extra-substituters).
+
+  - `trusted-binary-caches`  
+    *Deprecated:* `trusted-binary-caches` is now an alias to
+    [varlistentry\_title](#conf-trusted-substituters).
diff --git a/doc/manual/src/command-ref/files.md b/doc/manual/src/command-ref/files.md
new file mode 100644
index 000000000..df5646c05
--- /dev/null
+++ b/doc/manual/src/command-ref/files.md
@@ -0,0 +1,4 @@
+# Files
+
+This section lists configuration files that you can use when you work
+with Nix.

From 504b7abc452936b3118a18efea05026083ed8cdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 12:58:42 +0200
Subject: [PATCH 028/882] Convert commands

---
 doc/manual/src/SUMMARY.md                     |  11 +
 doc/manual/src/command-ref/command-ref.md     |   6 +-
 doc/manual/src/command-ref/main-commands.md   |   4 +
 doc/manual/src/command-ref/nix-build.md       | 130 +++
 doc/manual/src/command-ref/nix-channel.md     | 116 +++
 .../src/command-ref/nix-collect-garbage.md    |  46 +
 doc/manual/src/command-ref/nix-daemon.md      |  17 +
 doc/manual/src/command-ref/nix-env.md         | 926 +++++++++++++++++
 doc/manual/src/command-ref/nix-hash.md        | 120 +++
 doc/manual/src/command-ref/nix-instantiate.md | 184 ++++
 .../src/command-ref/nix-prefetch-url.md       |  87 ++
 doc/manual/src/command-ref/nix-shell.md       | 291 ++++++
 doc/manual/src/command-ref/nix-store.md       | 954 ++++++++++++++++++
 doc/manual/src/command-ref/opt-common-syn.md  |  57 ++
 doc/manual/src/command-ref/opt-inst-syn.md    |  15 +
 15 files changed, 2960 insertions(+), 4 deletions(-)
 create mode 100644 doc/manual/src/command-ref/main-commands.md
 create mode 100644 doc/manual/src/command-ref/nix-build.md
 create mode 100644 doc/manual/src/command-ref/nix-channel.md
 create mode 100644 doc/manual/src/command-ref/nix-collect-garbage.md
 create mode 100644 doc/manual/src/command-ref/nix-daemon.md
 create mode 100644 doc/manual/src/command-ref/nix-env.md
 create mode 100644 doc/manual/src/command-ref/nix-hash.md
 create mode 100644 doc/manual/src/command-ref/nix-instantiate.md
 create mode 100644 doc/manual/src/command-ref/nix-prefetch-url.md
 create mode 100644 doc/manual/src/command-ref/nix-shell.md
 create mode 100644 doc/manual/src/command-ref/nix-store.md
 create mode 100644 doc/manual/src/command-ref/opt-common-syn.md
 create mode 100644 doc/manual/src/command-ref/opt-inst-syn.md

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index e8baf489b..80a41f8d7 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -47,8 +47,19 @@
 - [Command Reference](command-ref/command-ref.md)
   - [Common Options](command-ref/opt-common.md)
   - [Common Environment Variables](command-ref/env-common.md)
+  - [Main Commands](command-ref/main-commands.md)
+    - [nix-env](command-ref/nix-env.md)
+    - [nix-build](command-ref/nix-build.md)
+    - [nix-shell](command-ref/nix-shell.md)
+    - [nix-store](command-ref/nix-store.md)
   - [Utilities](command-ref/utilities.md)
+    - [nix-channel](command-ref/nix-channel.md)
+    - [nix-collect-garbage](command-ref/nix-collect-garbage.md)
     - [nix-copy-closure](command-ref/nix-copy-closure.md)
+    - [nix-daemon](command-ref/nix-daemon.md)
+    - [nix-hash](command-ref/nix-hash.md)
+    - [nix-instantiate](command-ref/nix-instantiate.md)
+    - [nix-prefetch-url](command-ref/nix-prefetch-url.md)
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
 - [Glossary](glossary.md)
diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/command-ref.md
index 2c7dbf22e..6a78075db 100644
--- a/doc/manual/src/command-ref/command-ref.md
+++ b/doc/manual/src/command-ref/command-ref.md
@@ -1,4 +1,2 @@
-# Command Reference
-
-This section lists commands and options that you can use when you
-work with Nix.
+This section lists commands and options that you can use when you work
+with Nix.
diff --git a/doc/manual/src/command-ref/main-commands.md b/doc/manual/src/command-ref/main-commands.md
new file mode 100644
index 000000000..e4f1f1d0e
--- /dev/null
+++ b/doc/manual/src/command-ref/main-commands.md
@@ -0,0 +1,4 @@
+# Main Commands
+
+This section lists commands and options that you can use when you work
+with Nix.
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
new file mode 100644
index 000000000..7d0567760
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -0,0 +1,130 @@
+nix-build
+
+1
+
+Nix
+
+nix-build
+
+build a Nix expression
+
+nix-build
+
+\--arg
+
+name
+
+value
+
+\--argstr
+
+name
+
+value
+
+\--attr
+
+\-A
+
+attrPath
+
+\--no-out-link
+
+\--dry-run
+
+\--out-link
+
+\-o
+
+outlink
+
+paths
+
+# Description
+
+The `nix-build` command builds the derivations described by the Nix
+expressions in paths. If the build succeeds, it places a symlink to the
+result in the current directory. The symlink is called `result`. If
+there are multiple Nix expressions, or the Nix expressions evaluate to
+multiple derivations, multiple sequentially numbered symlinks are
+created (`result`, `result-2`, and so on).
+
+If no paths are specified, then `nix-build` will use `default.nix` in
+the current directory, if it exists.
+
+If an element of paths starts with `http://` or `https://`, it is
+interpreted as the URL of a tarball that will be downloaded and unpacked
+to a temporary location. The tarball must include a single top-level
+directory containing at least a file named `default.nix`.
+
+`nix-build` is essentially a wrapper around
+[`nix-instantiate`](#sec-nix-instantiate) (to translate a high-level Nix
+expression to a low-level store derivation) and [`nix-store
+--realise`](#rsec-nix-store-realise) (to build the store derivation).
+
+> **Warning**
+> 
+> The result of the build is automatically registered as a root of the
+> Nix garbage collector. This root disappears automatically when the
+> `result` symlink is deleted or renamed. So don’t rename the symlink.
+
+# Options
+
+All options not listed here are passed to `nix-store
+--realise`, except for `--arg` and `--attr` / `-A` which are passed to
+`nix-instantiate`. See also [???](#sec-common-options).
+
+  - `--no-out-link`  
+    Do not create a symlink to the output path. Note that as a result
+    the output does not become a root of the garbage collector, and so
+    might be deleted by `nix-store
+                    --gc`.
+
+  - `--dry-run`  
+    Show what store paths would be built or downloaded.
+
+  - `--out-link` / `-o` outlink  
+    Change the name of the symlink to the output path created from
+    `result` to outlink.
+
+The following common options are supported:
+
+# Examples
+
+    $ nix-build '<nixpkgs>' -A firefox
+    store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
+    /nix/store/d18hyl92g30l...-firefox-1.5.0.7
+    
+    $ ls -l result
+    lrwxrwxrwx  ...  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
+    
+    $ ls ./result/bin/
+    firefox  firefox-config
+
+If a derivation has multiple outputs, `nix-build` will build the default
+(first) output. You can also build all outputs:
+
+    $ nix-build '<nixpkgs>' -A openssl.all
+
+This will create a symlink for each output named `result-outputname`.
+The suffix is omitted if the output name is `out`. So if `openssl` has
+outputs `out`, `bin` and `man`, `nix-build` will create symlinks
+`result`, `result-bin` and `result-man`. It’s also possible to build a
+specific output:
+
+    $ nix-build '<nixpkgs>' -A openssl.man
+
+This will create a symlink `result-man`.
+
+Build a Nix expression given on the command line:
+
+    $ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"'
+    $ cat ./result
+    bar
+
+Build the GNU Hello package from the latest revision of the master
+branch of Nixpkgs:
+
+    $ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
+
+# Environment variables
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
new file mode 100644
index 000000000..d427151a0
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -0,0 +1,116 @@
+nix-channel
+
+1
+
+Nix
+
+nix-channel
+
+manage Nix channels
+
+nix-channel
+
+\--add
+
+url
+
+name
+
+\--remove
+
+name
+
+\--list
+
+\--update
+
+names
+
+\--rollback
+
+generation
+
+# Description
+
+A Nix channel is a mechanism that allows you to automatically stay
+up-to-date with a set of pre-built Nix expressions. A Nix channel is
+just a URL that points to a place containing a set of Nix expressions.
+See also [???](#sec-channels).
+
+To see the list of official NixOS channels, visit
+<https://nixos.org/channels>.
+
+This command has the following operations:
+
+  - `--add` url \[name\]  
+    Adds a channel named name with URL url to the list of subscribed
+    channels. If name is omitted, it defaults to the last component of
+    url, with the suffixes `-stable` or `-unstable` removed.
+
+  - `--remove` name  
+    Removes the channel named name from the list of subscribed channels.
+
+  - `--list`  
+    Prints the names and URLs of all subscribed channels on standard
+    output.
+
+  - `--update` \[names…\]  
+    Downloads the Nix expressions of all subscribed channels (or only
+    those included in names if specified) and makes them the default for
+    `nix-env` operations (by symlinking them from the directory
+    `~/.nix-defexpr`).
+
+  - `--rollback` \[generation\]  
+    Reverts the previous call to `nix-channel
+                    --update`. Optionally, you can specify a specific channel generation
+    number to restore.
+
+Note that `--add` does not automatically perform an update.
+
+The list of subscribed channels is stored in `~/.nix-channels`.
+
+# Examples
+
+To subscribe to the Nixpkgs channel and install the GNU Hello package:
+
+    $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    $ nix-channel --update
+    $ nix-env -iA nixpkgs.hello
+
+You can revert channel updates using `--rollback`:
+
+    $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+    "14.04.527.0e935f1"
+    
+    $ nix-channel --rollback
+    switching from generation 483 to 482
+    
+    $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+    "14.04.526.dbadfad"
+
+# Files
+
+  - `/nix/var/nix/profiles/per-user/username/channels`  
+    `nix-channel` uses a `nix-env` profile to keep track of previous
+    versions of the subscribed channels. Every time you run `nix-channel
+    --update`, a new channel generation (that is, a symlink to the
+    channel Nix expressions in the Nix store) is created. This enables
+    `nix-channel --rollback` to revert to previous versions.
+
+  - `~/.nix-defexpr/channels`  
+    This is a symlink to
+    `/nix/var/nix/profiles/per-user/username/channels`. It ensures that
+    `nix-env` can find your channels. In a multi-user installation, you
+    may also have `~/.nix-defexpr/channels_root`, which links to the
+    channels of the root user.
+
+# Channel format
+
+A channel URL should point to a directory containing the following
+files:
+
+  - `nixexprs.tar.xz`  
+    A tarball containing Nix expressions and files referenced by them
+    (such as build scripts and patches). At the top level, the tarball
+    should contain a single directory. That directory must contain a
+    file `default.nix` that serves as the channel’s “entry point”.
diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
new file mode 100644
index 000000000..7946dc875
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -0,0 +1,46 @@
+nix-collect-garbage
+
+1
+
+Nix
+
+nix-collect-garbage
+
+delete unreachable store paths
+
+nix-collect-garbage
+
+\--delete-old
+
+\-d
+
+\--delete-older-than
+
+period
+
+\--max-freed
+
+bytes
+
+\--dry-run
+
+# Description
+
+The command `nix-collect-garbage` is mostly an alias of [`nix-store
+--gc`](#rsec-nix-store-gc), that is, it deletes all unreachable paths in
+the Nix store to clean up your system. However, it provides two
+additional options: `-d` (`--delete-old`), which deletes all old
+generations of all profiles in `/nix/var/nix/profiles` by invoking
+`nix-env --delete-generations old` on all profiles (of course, this
+makes rollbacks to previous configurations impossible); and
+`--delete-older-than` period, where period is a value such as `30d`,
+which deletes all generations older than the specified number of days in
+all profiles in `/nix/var/nix/profiles` (except for the generations that
+were active at that point in time).
+
+# Example
+
+To delete from the Nix store everything that is not used by the current
+generations of each profile, do
+
+    $ nix-collect-garbage -d
diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/src/command-ref/nix-daemon.md
new file mode 100644
index 000000000..b0570789c
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-daemon.md
@@ -0,0 +1,17 @@
+nix-daemon
+
+8
+
+Nix
+
+nix-daemon
+
+Nix multi-user support daemon
+
+nix-daemon
+
+# Description
+
+The Nix daemon is necessary in multi-user Nix installations. It performs
+build actions and other operations on the Nix store on behalf of
+unprivileged users.
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
new file mode 100644
index 000000000..90bf6ea08
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -0,0 +1,926 @@
+nix-env
+
+1
+
+Nix
+
+nix-env
+
+manipulate or query Nix user environments
+
+nix-env
+
+\--arg
+
+name
+
+value
+
+\--argstr
+
+name
+
+value
+
+\--file
+
+\-f
+
+path
+
+\--profile
+
+\-p
+
+path
+
+\--system-filter
+
+system
+
+\--dry-run
+
+operation
+
+options
+
+arguments
+
+# Description
+
+The command `nix-env` is used to manipulate Nix user environments. User
+environments are sets of software packages available to a user at some
+point in time. In other words, they are a synthesised view of the
+programs available in the Nix store. There may be many user
+environments: different users can have different environments, and
+individual users can switch between different environments.
+
+`nix-env` takes exactly one *operation* flag which indicates the
+subcommand to be performed. These are documented below.
+
+# Selectors
+
+Several commands, such as `nix-env -q` and `nix-env -i`, take a list of
+arguments that specify the packages on which to operate. These are
+extended regular expressions that must match the entire name of the
+package. (For details on regular expressions, see regex7.) The match is
+case-sensitive. The regular expression can optionally be followed by a
+dash and a version number; if omitted, any version of the package will
+match. Here are some examples:
+
+  - `firefox`  
+    Matches the package name `firefox` and any version.
+
+  - `firefox-32.0`  
+    Matches the package name `firefox` and version `32.0`.
+
+  - `gtk\\+`  
+    Matches the package name `gtk+`. The `+` character must be escaped
+    using a backslash to prevent it from being interpreted as a
+    quantifier, and the backslash must be escaped in turn with another
+    backslash to ensure that the shell passes it on.
+
+  - `.\*`  
+    Matches any package name. This is the default for most commands.
+
+  - `'.*zip.*'`  
+    Matches any package name containing the string `zip`. Note the dots:
+    `'*zip*'` does not work, because in a regular expression, the
+    character `*` is interpreted as a quantifier.
+
+  - `'.*(firefox|chromium).*'`  
+    Matches any package name containing the strings `firefox` or
+    `chromium`.
+
+# Common options
+
+This section lists the options that are common to all operations. These
+options are allowed for every subcommand, though they may not always
+have an effect. See also [???](#sec-common-options).
+
+  - `--file` / `-f` path  
+    Specifies the Nix expression (designated below as the *active Nix
+    expression*) used by the `--install`, `--upgrade`, and `--query
+    --available` operations to obtain derivations. The default is
+    `~/.nix-defexpr`.
+    
+    If the argument starts with `http://` or `https://`, it is
+    interpreted as the URL of a tarball that will be downloaded and
+    unpacked to a temporary location. The tarball must include a single
+    top-level directory containing at least a file named `default.nix`.
+
+  - `--profile` / `-p` path  
+    Specifies the profile to be used by those operations that operate on
+    a profile (designated below as the *active profile*). A profile is a
+    sequence of user environments called *generations*, one of which is
+    the *current generation*.
+
+  - `--dry-run`  
+    For the `--install`, `--upgrade`, `--uninstall`,
+    `--switch-generation`, `--delete-generations` and `--rollback`
+    operations, this flag will cause `nix-env` to print what *would* be
+    done if this flag had not been specified, without actually doing it.
+    
+    `--dry-run` also prints out which paths will be
+    [substituted](#gloss-substitute) (i.e., downloaded) and which paths
+    will be built from source (because no substitute is available).
+
+  - `--system-filter` system  
+    By default, operations such as `--query
+                    --available` show derivations matching any platform. This option
+    allows you to use derivations for the specified platform system.
+
+<!-- end list -->
+
+# Files
+
+  - `~/.nix-defexpr`  
+    The source for the default Nix expressions used by the `--install`,
+    `--upgrade`, and `--query
+                    --available` operations to obtain derivations. The `--file` option
+    may be used to override this default.
+    
+    If `~/.nix-defexpr` is a file, it is loaded as a Nix expression. If
+    the expression is a set, it is used as the default Nix expression.
+    If the expression is a function, an empty set is passed as argument
+    and the return value is used as the default Nix expression.
+    
+    If `~/.nix-defexpr` is a directory containing a `default.nix` file,
+    that file is loaded as in the above paragraph.
+    
+    If `~/.nix-defexpr` is a directory without a `default.nix` file,
+    then its contents (both files and subdirectories) are loaded as Nix
+    expressions. The expressions are combined into a single set, each
+    expression under an attribute with the same name as the original
+    file or subdirectory.
+    
+    For example, if `~/.nix-defexpr` contains two files, `foo.nix` and
+    `bar.nix`, then the default Nix expression will essentially be
+    
+        {
+          foo = import ~/.nix-defexpr/foo.nix;
+          bar = import ~/.nix-defexpr/bar.nix;
+        }
+    
+    The file `manifest.nix` is always ignored. Subdirectories without a
+    `default.nix` file are traversed recursively in search of more Nix
+    expressions, but the names of these intermediate directories are not
+    added to the attribute paths of the default Nix expression.
+    
+    The command `nix-channel` places symlinks to the downloaded Nix
+    expressions from each subscribed channel in this directory.
+
+  - `~/.nix-profile`  
+    A symbolic link to the user's current profile. By default, this
+    symlink points to `prefix/var/nix/profiles/default`. The `PATH`
+    environment variable should include `~/.nix-profile/bin` for the
+    user environment to be visible to the user.
+
+# Operation `--install`
+
+## Synopsis
+
+nix-env
+
+\--install
+
+\-i
+
+\--preserve-installed
+
+\-P
+
+\--remove-all
+
+\-r
+
+args
+
+## Description
+
+The install operation creates a new user environment, based on the
+current generation of the active profile, to which a set of store paths
+described by args is added. The arguments args map to store paths in a
+number of possible ways:
+
+  - By default, args is a set of derivation names denoting derivations
+    in the active Nix expression. These are realised, and the resulting
+    output paths are installed. Currently installed derivations with a
+    name equal to the name of a derivation being added are removed
+    unless the option `--preserve-installed` is specified.
+    
+    If there are multiple derivations matching a name in args that have
+    the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the
+    derivation with the highest *priority* is used. A derivation can
+    define a priority by declaring the `meta.priority` attribute. This
+    attribute should be a number, with a higher value denoting a lower
+    priority. The default priority is `0`.
+    
+    If there are multiple matching derivations with the same priority,
+    then the derivation with the highest version will be installed.
+    
+    You can force the installation of multiple derivations with the same
+    name by being specific about the versions. For instance, `nix-env -i
+    gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will
+    probably cause a user environment conflict\!).
+
+  - If [`--attr`](#opt-attr) (`-A`) is specified, the arguments are
+    *attribute paths* that select attributes from the top-level Nix
+    expression. This is faster than using derivation names and
+    unambiguous. To find out the attribute paths of available packages,
+    use `nix-env -qaP`.
+
+  - If `--from-profile` path is given, args is a set of names denoting
+    installed store paths in the profile path. This is an easy way to
+    copy user environment elements from one profile to another.
+
+  - If `--from-expression` is given, args are Nix
+    [functions](#ss-functions) that are called with the active Nix
+    expression as their single argument. The derivations returned by
+    those function calls are installed. This allows derivations to be
+    specified in an unambiguous way, which is necessary if there are
+    multiple derivations with the same name.
+
+  - If args are store derivations, then these are
+    [realised](#rsec-nix-store-realise), and the resulting output paths
+    are installed.
+
+  - If args are store paths that are not store derivations, then these
+    are [realised](#rsec-nix-store-realise) and installed.
+
+  - By default all outputs are installed for each derivation. That can
+    be reduced by setting `meta.outputsToInstall`.
+
+## Flags
+
+  - `--prebuilt-only` / `-b`  
+    Use only derivations for which a substitute is registered, i.e.,
+    there is a pre-built binary available that can be downloaded in lieu
+    of building the derivation. Thus, no packages will be built from
+    source.
+
+  - `--preserve-installed`; `-P`  
+    Do not remove derivations with a name matching one of the
+    derivations being installed. Usually, trying to have two versions of
+    the same package installed in the same generation of a profile will
+    lead to an error in building the generation, due to file name
+    clashes between the two versions. However, this is not the case for
+    all packages.
+
+  - `--remove-all`; `-r`  
+    Remove all previously installed packages first. This is equivalent
+    to running `nix-env -e '.*'` first, except that everything happens
+    in a single transaction.
+
+## Examples
+
+To install a specific version of `gcc` from the active Nix expression:
+
+    $ nix-env --install gcc-3.3.2
+    installing `gcc-3.3.2'
+    uninstalling `gcc-3.1'
+
+Note the previously installed version is removed, since
+`--preserve-installed` was not specified.
+
+To install an arbitrary version:
+
+    $ nix-env --install gcc
+    installing `gcc-3.3.2'
+
+To install using a specific attribute:
+
+    $ nix-env -i -A gcc40mips
+    $ nix-env -i -A xorg.xorgserver
+
+To install all derivations in the Nix expression `foo.nix`:
+
+    $ nix-env -f ~/foo.nix -i '.*'
+
+To copy the store path with symbolic name `gcc` from another profile:
+
+    $ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc
+
+To install a specific store derivation (typically created by
+`nix-instantiate`):
+
+    $ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
+
+To install a specific output path:
+
+    $ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3
+
+To install from a Nix expression specified on the command-line:
+
+    $ nix-env -f ./foo.nix -i -E \
+        'f: (f {system = "i686-linux";}).subversionWithJava'
+
+I.e., this evaluates to `(f: (f {system =
+"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting
+the `subversionWithJava` attribute from the set returned by calling the
+function defined in `./foo.nix`.
+
+A dry-run tells you which paths will be downloaded or built from source:
+
+    $ nix-env -f '<nixpkgs>' -iA hello --dry-run
+    (dry run; not doing anything)
+    installing ‘hello-2.10’
+    this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+      /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
+      ...
+
+To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12
+channel:
+
+    $ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
+
+# Operation `--upgrade`
+
+## Synopsis
+
+nix-env
+
+\--upgrade
+
+\-u
+
+\--lt
+
+\--leq
+
+\--eq
+
+\--always
+
+args
+
+## Description
+
+The upgrade operation creates a new user environment, based on the
+current generation of the active profile, in which all store paths are
+replaced for which there are newer versions in the set of paths
+described by args. Paths for which there are no newer versions are left
+untouched; this is not an error. It is also not an error if an element
+of args matches no installed derivations.
+
+For a description of how args is mapped to a set of store paths, see
+[`--install`](#rsec-nix-env-install). If args describes multiple store
+paths with the same symbolic name, only the one with the highest version
+is installed.
+
+## Flags
+
+  - `--lt`  
+    Only upgrade a derivation to newer versions. This is the default.
+
+  - `--leq`  
+    In addition to upgrading to newer versions, also “upgrade” to
+    derivations that have the same version. Version are not a unique
+    identification of a derivation, so there may be many derivations
+    that have the same version. This flag may be useful to force
+    “synchronisation” between the installed and available derivations.
+
+  - `--eq`  
+    *Only* “upgrade” to derivations that have the same version. This may
+    not seem very useful, but it actually is, e.g., when there is a new
+    release of Nixpkgs and you want to replace installed applications
+    with the same versions built against newer dependencies (to reduce
+    the number of dependencies floating around on your system).
+
+  - `--always`  
+    In addition to upgrading to newer versions, also “upgrade” to
+    derivations that have the same or a lower version. I.e., derivations
+    may actually be downgraded depending on what is available in the
+    active Nix expression.
+
+For the other flags, see `--install`.
+
+## Examples
+
+    $ nix-env --upgrade gcc
+    upgrading `gcc-3.3.1' to `gcc-3.4'
+    
+    $ nix-env -u gcc-3.3.2 --always (switch to a specific version)
+    upgrading `gcc-3.4' to `gcc-3.3.2'
+    
+    $ nix-env --upgrade pan
+    (no upgrades available, so nothing happens)
+    
+    $ nix-env -u (try to upgrade everything)
+    upgrading `hello-2.1.2' to `hello-2.1.3'
+    upgrading `mozilla-1.2' to `mozilla-1.4'
+
+## Versions
+
+The upgrade operation determines whether a derivation `y` is an upgrade
+of a derivation `x` by looking at their respective `name` attributes.
+The names (e.g., `gcc-3.3.1` are split into two parts: the package name
+(`gcc`), and the version (`3.3.1`). The version part starts after the
+first dash not followed by a letter. `x` is considered an upgrade of `y`
+if their package names match, and the version of `y` is higher that that
+of `x`.
+
+The versions are compared by splitting them into contiguous components
+of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1,
+"pre", 5]`. These lists are then compared lexicographically (from left
+to right). Corresponding components `a` and `b` are compared as follows.
+If they are both numbers, integer comparison is used. If `a` is an empty
+string and `b` is a number, `a` is considered less than `b`. The special
+string component `pre` (for *pre-release*) is considered to be less than
+other components. String components are considered less than number
+components. Otherwise, they are compared lexicographically (i.e., using
+case-sensitive string comparison).
+
+This is illustrated by the following examples:
+
+    1.0 < 2.3
+    2.1 < 2.3
+    2.3 = 2.3
+    2.5 > 2.3
+    3.1 > 2.3
+    2.3.1 > 2.3
+    2.3.1 > 2.3a
+    2.3pre1 < 2.3
+    2.3pre3 < 2.3pre12
+    2.3a < 2.3c
+    2.3pre1 < 2.3c
+    2.3pre1 < 2.3q
+
+# Operation `--uninstall`
+
+## Synopsis
+
+nix-env
+
+\--uninstall
+
+\-e
+
+drvnames
+
+## Description
+
+The uninstall operation creates a new user environment, based on the
+current generation of the active profile, from which the store paths
+designated by the symbolic names names are removed.
+
+## Examples
+
+    $ nix-env --uninstall gcc
+    $ nix-env -e '.*' (remove everything)
+
+# Operation `--set`
+
+## Synopsis
+
+nix-env
+
+\--set
+
+drvname
+
+## Description
+
+The `--set` operation modifies the current generation of a profile so
+that it contains exactly the specified derivation, and nothing else.
+
+## Examples
+
+The following updates a profile such that its current generation will
+contain just Firefox:
+
+    $ nix-env -p /nix/var/nix/profiles/browser --set firefox
+
+# Operation `--set-flag`
+
+## Synopsis
+
+nix-env
+
+\--set-flag
+
+name
+
+value
+
+drvnames
+
+## Description
+
+The `--set-flag` operation allows meta attributes of installed packages
+to be modified. There are several attributes that can be usefully
+modified, because they affect the behaviour of `nix-env` or the user
+environment build script:
+
+  - `priority` can be changed to resolve filename clashes. The user
+    environment build script uses the `meta.priority` attribute of
+    derivations to resolve filename collisions between packages. Lower
+    priority values denote a higher priority. For instance, the GCC
+    wrapper package and the Binutils package in Nixpkgs both have a file
+    `bin/ld`, so previously if you tried to install both you would get a
+    collision. Now, on the other hand, the GCC wrapper declares a higher
+    priority than Binutils, so the former’s `bin/ld` is symlinked in the
+    user environment.
+
+  - `keep` can be set to `true` to prevent the package from being
+    upgraded or replaced. This is useful if you want to hang on to an
+    older version of a package.
+
+  - `active` can be set to `false` to “disable” the package. That is, no
+    symlinks will be generated to the files of the package, but it
+    remains part of the profile (so it won’t be garbage-collected). It
+    can be set back to `true` to re-enable the package.
+
+## Examples
+
+To prevent the currently installed Firefox from being upgraded:
+
+    $ nix-env --set-flag keep true firefox
+
+After this, `nix-env -u` will ignore Firefox.
+
+To disable the currently installed Firefox, then install a new Firefox
+while the old remains part of the profile:
+
+    $ nix-env -q
+    firefox-2.0.0.9 (the current one)
+    
+    $ nix-env --preserve-installed -i firefox-2.0.0.11
+    installing `firefox-2.0.0.11'
+    building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
+    collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox'
+      and `/nix/store/...-firefox-2.0.0.9/bin/firefox'.
+    (i.e., can’t have two active at the same time)
+    
+    $ nix-env --set-flag active false firefox
+    setting flag on `firefox-2.0.0.9'
+    
+    $ nix-env --preserve-installed -i firefox-2.0.0.11
+    installing `firefox-2.0.0.11'
+    
+    $ nix-env -q
+    firefox-2.0.0.11 (the enabled one)
+    firefox-2.0.0.9 (the disabled one)
+
+To make files from `binutils` take precedence over files from `gcc`:
+
+    $ nix-env --set-flag priority 5 binutils
+    $ nix-env --set-flag priority 10 gcc
+
+# Operation `--query`
+
+## Synopsis
+
+nix-env
+
+\--query
+
+\-q
+
+\--installed
+
+\--available
+
+\-a
+
+\--status
+
+\-s
+
+\--attr-path
+
+\-P
+
+\--no-name
+
+\--compare-versions
+
+\-c
+
+\--system
+
+\--drv-path
+
+\--out-path
+
+\--description
+
+\--meta
+
+\--xml
+
+\--json
+
+\--prebuilt-only
+
+\-b
+
+\--attr
+
+\-A
+
+attribute-path
+
+names
+
+## Description
+
+The query operation displays information about either the store paths
+that are installed in the current generation of the active profile
+(`--installed`), or the derivations that are available for installation
+in the active Nix expression (`--available`). It only prints information
+about derivations whose symbolic name matches one of names.
+
+The derivations are sorted by their `name` attributes.
+
+## Source selection
+
+The following flags specify the set of things on which the query
+operates.
+
+  - `--installed`  
+    The query operates on the store paths that are installed in the
+    current generation of the active profile. This is the default.
+
+  - `--available`; `-a`  
+    The query operates on the derivations that are available in the
+    active Nix expression.
+
+## Queries
+
+The following flags specify what information to display about the
+selected derivations. Multiple flags may be specified, in which case the
+information is shown in the order given here. Note that the name of the
+derivation is shown unless `--no-name` is specified.
+
+  - `--xml`  
+    Print the result in an XML representation suitable for automatic
+    processing by other tools. The root element is called `items`, which
+    contains a `item` element for each available or installed
+    derivation. The fields discussed below are all stored in attributes
+    of the `item` elements.
+
+  - `--json`  
+    Print the result in a JSON representation suitable for automatic
+    processing by other tools.
+
+  - `--prebuilt-only` / `-b`  
+    Show only derivations for which a substitute is registered, i.e.,
+    there is a pre-built binary available that can be downloaded in lieu
+    of building the derivation. Thus, this shows all packages that
+    probably can be installed quickly.
+
+  - `--status`; `-s`  
+    Print the *status* of the derivation. The status consists of three
+    characters. The first is `I` or `-`, indicating whether the
+    derivation is currently installed in the current generation of the
+    active profile. This is by definition the case for `--installed`,
+    but not for `--available`. The second is `P` or `-`, indicating
+    whether the derivation is present on the system. This indicates
+    whether installation of an available derivation will require the
+    derivation to be built. The third is `S` or `-`, indicating whether
+    a substitute is available for the derivation.
+
+  - `--attr-path`; `-P`  
+    Print the *attribute path* of the derivation, which can be used to
+    unambiguously select it using the [`--attr` option](#opt-attr)
+    available in commands that install derivations like `nix-env
+    --install`. This option only works together with `--available`
+
+  - `--no-name`  
+    Suppress printing of the `name` attribute of each derivation.
+
+  - `--compare-versions` / `-c`  
+    Compare installed versions to available versions, or vice versa (if
+    `--available` is given). This is useful for quickly seeing whether
+    upgrades for installed packages are available in a Nix expression. A
+    column is added with the following meaning:
+    
+      - `<` version  
+        A newer version of the package is available or installed.
+    
+      - `=` version  
+        At most the same version of the package is available or
+        installed.
+    
+      - `>` version  
+        Only older versions of the package are available or installed.
+    
+      - `- ?`  
+        No version of the package is available or installed.
+
+  - `--system`  
+    Print the `system` attribute of the derivation.
+
+  - `--drv-path`  
+    Print the path of the store derivation.
+
+  - `--out-path`  
+    Print the output path of the derivation.
+
+  - `--description`  
+    Print a short (one-line) description of the derivation, if
+    available. The description is taken from the `meta.description`
+    attribute of the derivation.
+
+  - `--meta`  
+    Print all of the meta-attributes of the derivation. This option is
+    only available with `--xml` or `--json`.
+
+## Examples
+
+To show installed packages:
+
+    $ nix-env -q
+    bison-1.875c
+    docbook-xml-4.2
+    firefox-1.0.4
+    MPlayer-1.0pre7
+    ORBit2-2.8.3
+    …
+
+To show available packages:
+
+    $ nix-env -qa
+    firefox-1.0.7
+    GConf-2.4.0.1
+    MPlayer-1.0pre7
+    ORBit2-2.8.3
+    …
+
+To show the status of available packages:
+
+    $ nix-env -qas
+    -P- firefox-1.0.7   (not installed but present)
+    --S GConf-2.4.0.1   (not present, but there is a substitute for fast installation)
+    --S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!)
+    IP- ORBit2-2.8.3    (installed and by definition present)
+    …
+
+To show available packages in the Nix expression `foo.nix`:
+
+    $ nix-env -f ./foo.nix -qa
+    foo-1.2.3
+
+To compare installed versions to what’s available:
+
+    $ nix-env -qc
+    ...
+    acrobat-reader-7.0 - ?      (package is not available at all)
+    autoconf-2.59      = 2.59   (same version)
+    firefox-1.0.4      < 1.0.7  (a more recent version is available)
+    ...
+
+To show all packages with “`zip`” in the name:
+
+    $ nix-env -qa '.*zip.*'
+    bzip2-1.0.6
+    gzip-1.6
+    zip-3.0
+    …
+
+To show all packages with “`firefox`” or “`chromium`” in the name:
+
+    $ nix-env -qa '.*(firefox|chromium).*'
+    chromium-37.0.2062.94
+    chromium-beta-38.0.2125.24
+    firefox-32.0.3
+    firefox-with-plugins-13.0.1
+    …
+
+To show all packages in the latest revision of the Nixpkgs repository:
+
+    $ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
+
+# Operation `--switch-profile`
+
+## Synopsis
+
+nix-env
+
+\--switch-profile
+
+\-S
+
+path
+
+## Description
+
+This operation makes path the current profile for the user. That is, the
+symlink `~/.nix-profile` is made to point to path.
+
+## Examples
+
+    $ nix-env -S ~/my-profile
+
+# Operation `--list-generations`
+
+## Synopsis
+
+nix-env
+
+\--list-generations
+
+## Description
+
+This operation print a list of all the currently existing generations
+for the active profile. These may be switched to using the
+`--switch-generation` operation. It also prints the creation date of the
+generation, and indicates the current generation.
+
+## Examples
+
+    $ nix-env --list-generations
+      95   2004-02-06 11:48:24
+      96   2004-02-06 11:49:01
+      97   2004-02-06 16:22:45
+      98   2004-02-06 16:24:33   (current)
+
+# Operation `--delete-generations`
+
+## Synopsis
+
+nix-env
+
+\--delete-generations
+
+generations
+
+## Description
+
+This operation deletes the specified generations of the current profile.
+The generations can be a list of generation numbers, the special value
+`old` to delete all non-current generations, a value such as `30d` to
+delete all generations older than the specified number of days (except
+for the generation that was active at that point in time), or a value
+such as `+5` to keep the last `5` generations ignoring any newer than
+current, e.g., if `30` is the current generation `+5` will delete
+generation `25` and all older generations. Periodically deleting old
+generations is important to make garbage collection effective.
+
+## Examples
+
+    $ nix-env --delete-generations 3 4 8
+    
+    $ nix-env --delete-generations +5
+    
+    $ nix-env --delete-generations 30d
+    
+    $ nix-env -p other_profile --delete-generations old
+
+# Operation `--switch-generation`
+
+## Synopsis
+
+nix-env
+
+\--switch-generation
+
+\-G
+
+generation
+
+## Description
+
+This operation makes generation number generation the current generation
+of the active profile. That is, if the `profile` is the path to the
+active profile, then the symlink `profile` is made to point to
+`profile-generation-link`, which is in turn a symlink to the actual user
+environment in the Nix store.
+
+Switching will fail if the specified generation does not exist.
+
+## Examples
+
+    $ nix-env -G 42
+    switching from generation 50 to 42
+
+# Operation `--rollback`
+
+## Synopsis
+
+nix-env
+
+\--rollback
+
+## Description
+
+This operation switches to the “previous” generation of the active
+profile, that is, the highest numbered generation lower than the current
+generation, if it exists. It is just a convenience wrapper around
+`--list-generations` and `--switch-generation`.
+
+## Examples
+
+    $ nix-env --rollback
+    switching from generation 92 to 91
+    
+    $ nix-env --rollback
+    error: no generation older than the current (91) exists
+
+# Environment variables
+
+  - `NIX_PROFILE`  
+    Location of the Nix profile. Defaults to the target of the symlink
+    `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default`
+    otherwise.
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
new file mode 100644
index 000000000..d433cbc5b
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -0,0 +1,120 @@
+nix-hash
+
+1
+
+Nix
+
+nix-hash
+
+compute the cryptographic hash of a path
+
+nix-hash
+
+\--flat
+
+\--base32
+
+\--truncate
+
+\--type
+
+hashAlgo
+
+path
+
+nix-hash
+
+\--to-base16
+
+hash
+
+nix-hash
+
+\--to-base32
+
+hash
+
+# Description
+
+The command `nix-hash` computes the cryptographic hash of the contents
+of each path and prints it on standard output. By default, it computes
+an MD5 hash, but other hash algorithms are available as well. The hash
+is printed in hexadecimal. To generate the same hash as
+`nix-prefetch-url` you have to specify multiple arguments, see below for
+an example.
+
+The hash is computed over a *serialisation* of each path: a dump of the
+file system tree rooted at the path. This allows directories and
+symlinks to be hashed as well as regular files. The dump is in the *NAR
+format* produced by [`nix-store` `--dump`](#refsec-nix-store-dump).
+Thus, `nix-hash
+path` yields the same cryptographic hash as `nix-store --dump
+path | md5sum`.
+
+# Options
+
+  - `--flat`  
+    Print the cryptographic hash of the contents of each regular file
+    path. That is, do not compute the hash over the dump of path. The
+    result is identical to that produced by the GNU commands `md5sum`
+    and `sha1sum`.
+
+  - `--base32`  
+    Print the hash in a base-32 representation rather than hexadecimal.
+    This base-32 representation is more compact and can be used in Nix
+    expressions (such as in calls to `fetchurl`).
+
+  - `--truncate`  
+    Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits.
+
+  - `--type` hashAlgo  
+    Use the specified cryptographic hash algorithm, which can be one of
+    `md5`, `sha1`, and `sha256`.
+
+  - `--to-base16`  
+    Don’t hash anything, but convert the base-32 hash representation
+    hash to hexadecimal.
+
+  - `--to-base32`  
+    Don’t hash anything, but convert the hexadecimal hash representation
+    hash to base-32.
+
+# Examples
+
+Computing the same hash as `nix-prefetch-url`:
+
+    $ nix-prefetch-url file://<(echo test)
+    1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+    $ nix-hash --type sha256 --flat --base32 <(echo test)
+    1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+
+Computing hashes:
+
+    $ mkdir test
+    $ echo "hello" > test/world
+    
+    $ nix-hash test/ (MD5 hash; default)
+    8179d3caeff1869b5ba1744e5a245c04
+    
+    $ nix-store --dump test/ | md5sum (for comparison)
+    8179d3caeff1869b5ba1744e5a245c04  -
+    
+    $ nix-hash --type sha1 test/
+    e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
+    
+    $ nix-hash --type sha1 --base32 test/
+    nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
+    
+    $ nix-hash --type sha256 --flat test/
+    error: reading file `test/': Is a directory
+    
+    $ nix-hash --type sha256 --flat test/world
+    5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03
+
+Converting between hexadecimal and base-32:
+
+    $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
+    nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
+    
+    $ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
+    e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
new file mode 100644
index 000000000..449f5f70f
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -0,0 +1,184 @@
+nix-instantiate
+
+1
+
+Nix
+
+nix-instantiate
+
+instantiate store derivations from Nix expressions
+
+nix-instantiate
+
+\--parse
+
+\--eval
+
+\--strict
+
+\--json
+
+\--xml
+
+\--read-write-mode
+
+\--arg
+
+name
+
+value
+
+\--attr
+
+\-A
+
+attrPath
+
+\--add-root
+
+path
+
+\--indirect
+
+\--expr
+
+\-E
+
+files
+
+nix-instantiate
+
+\--find-file
+
+files
+
+# Description
+
+The command `nix-instantiate` generates [store
+derivations](#gloss-derivation) from (high-level) Nix expressions. It
+evaluates the Nix expressions in each of files (which defaults to
+./default.nix). Each top-level expression should evaluate to a
+derivation, a list of derivations, or a set of derivations. The paths of
+the resulting store derivations are printed on standard output.
+
+If files is the character `-`, then a Nix expression will be read from
+standard input.
+
+See also [???](#sec-common-options) for a list of common options.
+
+# Options
+
+  - `--add-root` path; `--indirect`  
+    See the [corresponding options](#opt-add-root) in `nix-store`.
+
+  - `--parse`  
+    Just parse the input files, and print their abstract syntax trees on
+    standard output in ATerm format.
+
+  - `--eval`  
+    Just parse and evaluate the input files, and print the resulting
+    values on standard output. No instantiation of store derivations
+    takes place.
+
+  - `--find-file`  
+    Look up the given files in Nix’s search path (as specified by the
+    NIX\_PATH\</literal\> environment variable). If found, print the
+    corresponding absolute paths on standard output. For instance, if
+    `NIX_PATH` is `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate
+    --find-file nixpkgs/default.nix` will print
+    `/home/alice/nixpkgs/default.nix`.
+
+  - `--strict`  
+    When used with `--eval`, recursively evaluate list elements and
+    attributes. Normally, such sub-expressions are left unevaluated
+    (since the Nix expression language is lazy).
+    
+    > **Warning**
+    > 
+    > This option can cause non-termination, because lazy data
+    > structures can be infinitely large.
+
+  - `--json`  
+    When used with `--eval`, print the resulting value as an JSON
+    representation of the abstract syntax tree rather than as an ATerm.
+
+  - `--xml`  
+    When used with `--eval`, print the resulting value as an XML
+    representation of the abstract syntax tree rather than as an ATerm.
+    The schema is the same as that used by the [`toXML`
+    built-in](#builtin-toXML).
+
+  - `--read-write-mode`  
+    When used with `--eval`, perform evaluation in read/write mode so
+    nix language features that require it will still work (at the cost
+    of needing to do instantiation of every evaluated derivation). If
+    this option is not enabled, there may be uninstantiated store paths
+    in the final output.
+
+<!-- end list -->
+
+# Examples
+
+Instantiating store derivations from a Nix expression, and building them
+using `nix-store`:
+
+    $ nix-instantiate test.nix (instantiate)
+    /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
+    
+    $ nix-store -r $(nix-instantiate test.nix) (build)
+    ...
+    /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path)
+    
+    $ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
+    dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
+    ...
+
+You can also give a Nix expression on the command line:
+
+    $ nix-instantiate -E 'with import <nixpkgs> { }; hello'
+    /nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv
+
+This is equivalent to:
+
+    $ nix-instantiate '<nixpkgs>' -A hello
+
+Parsing and evaluating Nix expressions:
+
+    $ nix-instantiate --parse -E '1 + 2'
+    1 + 2
+    
+    $ nix-instantiate --eval -E '1 + 2'
+    3
+    
+    $ nix-instantiate --eval --xml -E '1 + 2'
+    <?xml version='1.0' encoding='utf-8'?>
+    <expr>
+      <int value="3" />
+    </expr>
+
+The difference between non-strict and strict evaluation:
+
+    $ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }'
+    ...
+      <attr name="x">
+        <string value="foo" />
+      </attr>
+      <attr name="y">
+        <unevaluated />
+      </attr>
+    ...
+
+Note that `y` is left unevaluated (the XML representation doesn’t
+attempt to show non-normal forms).
+
+    $ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }'
+    ...
+      <attr name="x">
+        <string value="foo" />
+      </attr>
+      <attr name="y">
+        <string value="foo" />
+      </attr>
+    ...
+
+# Environment variables
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
new file mode 100644
index 000000000..bf8084e45
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -0,0 +1,87 @@
+nix-prefetch-url
+
+1
+
+Nix
+
+nix-prefetch-url
+
+copy a file from a URL into the store and print its hash
+
+nix-prefetch-url
+
+\--version
+
+\--type
+
+hashAlgo
+
+\--print-path
+
+\--unpack
+
+\--name
+
+name
+
+url
+
+hash
+
+# Description
+
+The command `nix-prefetch-url` downloads the file referenced by the URL
+url, prints its cryptographic hash, and copies it into the Nix store.
+The file name in the store is `hash-baseName`, where baseName is
+everything following the final slash in url.
+
+This command is just a convenience for Nix expression writers. Often a
+Nix expression fetches some source distribution from the network using
+the `fetchurl` expression contained in Nixpkgs. However, `fetchurl`
+requires a cryptographic hash. If you don't know the hash, you would
+have to download the file first, and then `fetchurl` would download it
+again when you build your Nix expression. Since `fetchurl` uses the same
+name for the downloaded file as `nix-prefetch-url`, the redundant
+download can be avoided.
+
+If hash is specified, then a download is not performed if the Nix store
+already contains a file with the same hash and base name. Otherwise, the
+file is downloaded, and an error is signaled if the actual hash of the
+file does not match the specified hash.
+
+This command prints the hash on standard output. Additionally, if the
+option `--print-path` is used, the path of the downloaded file in the
+Nix store is also printed.
+
+# Options
+
+  - `--type` hashAlgo  
+    Use the specified cryptographic hash algorithm, which can be one of
+    `md5`, `sha1`, and `sha256`.
+
+  - `--print-path`  
+    Print the store path of the downloaded file on standard output.
+
+  - `--unpack`  
+    Unpack the archive (which must be a tarball or zip file) and add the
+    result to the Nix store. The resulting hash can be used with
+    functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`.
+
+  - `--name` name  
+    Override the name of the file in the Nix store. By default, this is
+    `hash-basename`, where basename is the last component of url.
+    Overriding the name is necessary when basename contains characters
+    that are not allowed in Nix store paths.
+
+# Examples
+
+    $ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
+    0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
+    
+    $ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
+    0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
+    /nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
+    
+    $ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz
+    079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7
+    /nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
new file mode 100644
index 000000000..c6910e3f5
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -0,0 +1,291 @@
+nix-shell
+
+1
+
+Nix
+
+nix-shell
+
+start an interactive shell based on a Nix expression
+
+nix-shell
+
+\--arg
+
+name
+
+value
+
+\--argstr
+
+name
+
+value
+
+\--attr
+
+\-A
+
+attrPath
+
+\--command
+
+cmd
+
+\--run
+
+cmd
+
+\--exclude
+
+regexp
+
+\--pure
+
+\--keep
+
+name
+
+\--packages
+
+\-p
+
+packages
+
+expressions
+
+path
+
+# Description
+
+The command `nix-shell` will build the dependencies of the specified
+derivation, but not the derivation itself. It will then start an
+interactive shell in which all environment variables defined by the
+derivation path have been set to their corresponding values, and the
+script `$stdenv/setup` has been sourced. This is useful for reproducing
+the environment of a derivation for development.
+
+If path is not given, `nix-shell` defaults to `shell.nix` if it exists,
+and `default.nix` otherwise.
+
+If path starts with `http://` or `https://`, it is interpreted as the
+URL of a tarball that will be downloaded and unpacked to a temporary
+location. The tarball must include a single top-level directory
+containing at least a file named `default.nix`.
+
+If the derivation defines the variable `shellHook`, it will be evaluated
+after `$stdenv/setup` has been sourced. Since this hook is not executed
+by regular Nix builds, it allows you to perform initialisation specific
+to `nix-shell`. For example, the derivation attribute
+
+    shellHook =
+      ''
+        echo "Hello shell"
+      '';
+
+will cause `nix-shell` to print `Hello shell`.
+
+# Options
+
+All options not listed here are passed to `nix-store
+--realise`, except for `--arg` and `--attr` / `-A` which are passed to
+`nix-instantiate`. See also [???](#sec-common-options).
+
+  - `--command` cmd  
+    In the environment of the derivation, run the shell command cmd.
+    This command is executed in an interactive shell. (Use `--run` to
+    use a non-interactive shell instead.) However, a call to `exit` is
+    implicitly added to the command, so the shell will exit after
+    running the command. To prevent this, add `return` at the end; e.g.
+    `--command
+                    "echo Hello; return"` will print `Hello` and then drop you into the
+    interactive shell. This can be useful for doing any additional
+    initialisation.
+
+  - `--run` cmd  
+    Like `--command`, but executes the command in a non-interactive
+    shell. This means (among other things) that if you hit Ctrl-C while
+    the command is running, the shell exits.
+
+  - `--exclude` regexp  
+    Do not build any dependencies whose store path matches the regular
+    expression regexp. This option may be specified multiple times.
+
+  - `--pure`  
+    If this flag is specified, the environment is almost entirely
+    cleared before the interactive shell is started, so you get an
+    environment that more closely corresponds to the “real” Nix build. A
+    few variables, in particular `HOME`, `USER` and `DISPLAY`, are
+    retained. Note that `~/.bashrc` and (depending on your Bash
+    installation) `/etc/bashrc` are still sourced, so any variables set
+    there will affect the interactive shell.
+
+  - `--packages` / `-p` packages…  
+    Set up an environment in which the specified packages are present.
+    The command line arguments are interpreted as attribute names inside
+    the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk`
+    will start a shell in which the packages denoted by the attribute
+    names `libjpeg` and `openjdk` are present.
+
+  - `-i` interpreter  
+    The chained script interpreter to be invoked by `nix-shell`. Only
+    applicable in `#!`-scripts (described
+    [below](#ssec-nix-shell-shebang)).
+
+  - `--keep` name  
+    When a `--pure` shell is started, keep the listed environment
+    variables.
+
+The following common options are supported:
+
+# Environment variables
+
+  - `NIX_BUILD_SHELL`  
+    Shell used to start the interactive environment. Defaults to the
+    `bash` found in `PATH`.
+
+# Examples
+
+To build the dependencies of the package Pan, and start an interactive
+shell in which to build it:
+
+    $ nix-shell '<nixpkgs>' -A pan
+    [nix-shell]$ unpackPhase
+    [nix-shell]$ cd pan-*
+    [nix-shell]$ configurePhase
+    [nix-shell]$ buildPhase
+    [nix-shell]$ ./pan/gui/pan
+
+To clear the environment first, and do some additional automatic
+initialisation of the interactive shell:
+
+    $ nix-shell '<nixpkgs>' -A pan --pure \
+        --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
+
+Nix expressions can also be given on the command line using the `-E` and
+`-p` flags. For instance, the following starts a shell containing the
+packages `sqlite` and `libX11`:
+
+    $ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
+
+A shorter way to do the same is:
+
+    $ nix-shell -p sqlite xorg.libX11
+    [nix-shell]$ echo $NIX_LDFLAGS
+    … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
+
+Note that `-p` accepts multiple full nix expressions that are valid in
+the `buildInputs = [ ... ]` shown above, not only package names. So the
+following is also legal:
+
+    $ nix-shell -p sqlite 'git.override { withManual = false; }'
+
+The `-p` flag looks up Nixpkgs in the Nix search path. You can override
+it by passing `-I` or setting `NIX_PATH`. For example, the following
+gives you a shell containing the Pan package from a specific revision of
+Nixpkgs:
+
+    $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+    
+    [nix-shell:~]$ pan --version
+    Pan 0.139
+
+# Use as a `#!`-interpreter
+
+You can use `nix-shell` as a script interpreter to allow scripts written
+in arbitrary languages to obtain their own dependencies via Nix. This is
+done by starting the script with the following lines:
+
+    #! /usr/bin/env nix-shell
+    #! nix-shell -i real-interpreter -p packages
+
+where real-interpreter is the “real” script interpreter that will be
+invoked by `nix-shell` after it has obtained the dependencies and
+initialised the environment, and packages are the attribute names of the
+dependencies in Nixpkgs.
+
+The lines starting with `#! nix-shell` specify `nix-shell` options (see
+above). Note that you cannot write `#! /usr/bin/env nix-shell -i ...`
+because many operating systems only allow one argument in `#!` lines.
+
+For example, here is a Python script that depends on Python and the
+`prettytable` package:
+
+    #! /usr/bin/env nix-shell
+    #! nix-shell -i python -p python pythonPackages.prettytable
+    
+    import prettytable
+    
+    # Print a simple table.
+    t = prettytable.PrettyTable(["N", "N^2"])
+    for n in range(1, 10): t.add_row([n, n * n])
+    print t
+
+Similarly, the following is a Perl script that specifies that it
+requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages:
+
+    #! /usr/bin/env nix-shell
+    #! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
+    
+    use HTML::TokeParser::Simple;
+    
+    # Fetch nixos.org and print all hrefs.
+    my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
+    
+    while (my $token = $p->get_tag("a")) {
+        my $href = $token->get_attr("href");
+        print "$href\n" if $href;
+    }
+
+Sometimes you need to pass a simple Nix expression to customize a
+package like Terraform:
+
+    #! /usr/bin/env nix-shell
+    #! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
+    
+    terraform apply
+
+> **Note**
+> 
+> You must use double quotes (`"`) when passing a simple Nix expression
+> in a nix-shell shebang.
+
+Finally, using the merging of multiple nix-shell shebangs the following
+Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable
+branch):
+
+    #! /usr/bin/env nix-shell
+    #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
+    #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+    
+    import Network.HTTP
+    import Text.HTML.TagSoup
+    
+    -- Fetch nixos.org and print all hrefs.
+    main = do
+      resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
+      body <- getResponseBody resp
+      let tags = filter (isTagOpenName "a") $ parseTags body
+      let tags' = map (fromAttrib "href") tags
+      mapM_ putStrLn $ filter (/= "") tags'
+
+If you want to be even more precise, you can specify a specific revision
+of Nixpkgs:
+
+    #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+
+The examples above all used `-p` to get dependencies from Nixpkgs. You
+can also use a Nix expression to build your own dependencies. For
+example, the Python example could have been written as:
+
+    #! /usr/bin/env nix-shell
+    #! nix-shell deps.nix -i python
+
+where the file `deps.nix` in the same directory as the `#!`-script
+contains:
+
+    with import <nixpkgs> {};
+    
+    runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
+
+# Environment variables
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
new file mode 100644
index 000000000..703e71076
--- /dev/null
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -0,0 +1,954 @@
+nix-store
+
+1
+
+Nix
+
+nix-store
+
+manipulate or query the Nix store
+
+nix-store
+
+\--add-root
+
+path
+
+\--indirect
+
+operation
+
+options
+
+arguments
+
+# Description
+
+The command `nix-store` performs primitive operations on the Nix store.
+You generally do not need to run this command manually.
+
+`nix-store` takes exactly one *operation* flag which indicates the
+subcommand to be performed. These are documented below.
+
+# Common options
+
+This section lists the options that are common to all operations. These
+options are allowed for every subcommand, though they may not always
+have an effect. See also [???](#sec-common-options) for a list of common
+options.
+
+  - `--add-root` path  
+    Causes the result of a realisation (`--realise` and
+    `--force-realise`) to be registered as a root of the garbage
+    collector(see [???](#ssec-gc-roots)). The root is stored in path,
+    which must be inside a directory that is scanned for roots by the
+    garbage collector (i.e., typically in a subdirectory of
+    `/nix/var/nix/gcroots/`) *unless* the `--indirect` flag is used.
+    
+    If there are multiple results, then multiple symlinks will be
+    created by sequentially numbering symlinks beyond the first one
+    (e.g., `foo`, `foo-2`, `foo-3`, and so on).
+
+  - `--indirect`  
+    In conjunction with `--add-root`, this option allows roots to be
+    stored *outside* of the GC roots directory. This is useful for
+    commands such as `nix-build` that place a symlink to the build
+    result in the current directory; such a build result should not be
+    garbage-collected unless the symlink is removed.
+    
+    The `--indirect` flag causes a uniquely named symlink to path to be
+    stored in `/nix/var/nix/gcroots/auto/`. For instance,
+    
+        $ nix-store --add-root /home/eelco/bla/result --indirect -r ...
+        
+        $ ls -l /nix/var/nix/gcroots/auto
+        lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
+        
+        $ ls -l /home/eelco/bla/result
+        lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10
+    
+    Thus, when `/home/eelco/bla/result` is removed, the GC root in the
+    `auto` directory becomes a dangling symlink and will be ignored by
+    the collector.
+    
+    > **Warning**
+    > 
+    > Note that it is not possible to move or rename indirect GC roots,
+    > since the symlink in the `auto` directory will still point to the
+    > old location.
+
+<!-- end list -->
+
+# Operation `--realise`
+
+## Synopsis
+
+nix-store
+
+\--realise
+
+\-r
+
+paths
+
+\--dry-run
+
+## Description
+
+The operation `--realise` essentially “builds” the specified store
+paths. Realisation is a somewhat overloaded term:
+
+  - If the store path is a *derivation*, realisation ensures that the
+    output paths of the derivation are [valid](#gloss-validity) (i.e.,
+    the output path and its closure exist in the file system). This can
+    be done in several ways. First, it is possible that the outputs are
+    already valid, in which case we are done immediately. Otherwise,
+    there may be [substitutes](#gloss-substitute) that produce the
+    outputs (e.g., by downloading them). Finally, the outputs can be
+    produced by performing the build action described by the derivation.
+
+  - If the store path is not a derivation, realisation ensures that the
+    specified path is valid (i.e., it and its closure exist in the file
+    system). If the path is already valid, we are done immediately.
+    Otherwise, the path and any missing paths in its closure may be
+    produced through substitutes. If there are no (successful)
+    subsitutes, realisation fails.
+
+The output path of each derivation is printed on standard output. (For
+non-derivations argument, the argument itself is printed.)
+
+The following flags are available:
+
+  - `--dry-run`  
+    Print on standard error a description of what packages would be
+    built or downloaded, without actually performing the operation.
+
+  - `--ignore-unknown`  
+    If a non-derivation path does not have a substitute, then silently
+    ignore it.
+
+  - `--check`  
+    This option allows you to check whether a derivation is
+    deterministic. It rebuilds the specified derivation and checks
+    whether the result is bitwise-identical with the existing outputs,
+    printing an error if that’s not the case. The outputs of the
+    specified derivation must already exist. When used with `-K`, if an
+    output path is not identical to the corresponding output from the
+    previous build, the new output path is left in
+    `/nix/store/name.check.`
+    
+    See also the `build-repeat` configuration option, which repeats a
+    derivation a number of times and prevents its outputs from being
+    registered as “valid” in the Nix store unless they are identical.
+
+Special exit codes:
+
+  - `100`  
+    Generic build failure, the builder process returned with a non-zero
+    exit code.
+
+  - `101`  
+    Build timeout, the build was aborted because it did not complete
+    within the specified [`timeout`](#conf-timeout).
+
+  - `102`  
+    Hash mismatch, the build output was rejected because it does not
+    match the specified [`outputHash`](#fixed-output-drvs).
+
+  - `104`  
+    Not deterministic, the build succeeded in check mode but the
+    resulting output is not binary reproducable.
+
+With the `--keep-going` flag it's possible for multiple failures to
+occur, in this case the 1xx status codes are or combined using binary
+or.
+
+    1100100
+       ^^^^
+       |||`- timeout
+       ||`-- output hash mismatch
+       |`--- build failure
+       `---- not deterministic
+
+## Examples
+
+This operation is typically used to build store derivations produced by
+[`nix-instantiate`](#sec-nix-instantiate):
+
+    $ nix-store -r $(nix-instantiate ./test.nix)
+    /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
+
+This is essentially what [`nix-build`](#sec-nix-build) does.
+
+To test whether a previously-built derivation is deterministic:
+
+    $ nix-build '<nixpkgs>' -A hello --check -K
+
+# Operation `--serve`
+
+## Synopsis
+
+nix-store
+
+\--serve
+
+\--write
+
+## Description
+
+The operation `--serve` provides access to the Nix store over stdin and
+stdout, and is intended to be used as a means of providing Nix store
+access to a restricted ssh user.
+
+The following flags are available:
+
+  - `--write`  
+    Allow the connected client to request the realization of
+    derivations. In effect, this can be used to make the host act as a
+    remote builder.
+
+## Examples
+
+To turn a host into a build server, the `authorized_keys` file can be
+used to provide build access to a given SSH public key:
+
+    $ cat <<EOF >>/root/.ssh/authorized_keys
+    command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA...
+    EOF
+
+# Operation `--gc`
+
+## Synopsis
+
+nix-store
+
+\--gc
+
+\--print-roots
+
+\--print-live
+
+\--print-dead
+
+\--max-freed
+
+bytes
+
+## Description
+
+Without additional flags, the operation `--gc` performs a garbage
+collection on the Nix store. That is, all paths in the Nix store not
+reachable via file system references from a set of “roots”, are deleted.
+
+The following suboperations may be specified:
+
+  - `--print-roots`  
+    This operation prints on standard output the set of roots used by
+    the garbage collector. What constitutes a root is described in
+    [???](#ssec-gc-roots).
+
+  - `--print-live`  
+    This operation prints on standard output the set of “live” store
+    paths, which are all the store paths reachable from the roots. Live
+    paths should never be deleted, since that would break consistency —
+    it would become possible that applications are installed that
+    reference things that are no longer present in the store.
+
+  - `--print-dead`  
+    This operation prints out on standard output the set of “dead” store
+    paths, which is just the opposite of the set of live paths: any path
+    in the store that is not live (with respect to the roots) is dead.
+
+By default, all unreachable paths are deleted. The following options
+control what gets deleted and in what order:
+
+  - `--max-freed` bytes  
+    Keep deleting paths until at least bytes bytes have been deleted,
+    then stop. The argument bytes can be followed by the multiplicative
+    suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB or TiB units.
+
+The behaviour of the collector is also influenced by the
+[`keep-outputs`](#conf-keep-outputs) and
+[`keep-derivations`](#conf-keep-derivations) variables in the Nix
+configuration file.
+
+By default, the collector prints the total number of freed bytes when it
+finishes (or when it is interrupted). With `--print-dead`, it prints the
+number of bytes that would be freed.
+
+## Examples
+
+To delete all unreachable paths, just do:
+
+    $ nix-store --gc
+    deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
+    ...
+    8825586 bytes freed (8.42 MiB)
+
+To delete at least 100 MiBs of unreachable paths:
+
+    $ nix-store --gc --max-freed $((100 * 1024 * 1024))
+
+# Operation `--delete`
+
+## Synopsis
+
+nix-store
+
+\--delete
+
+\--ignore-liveness
+
+paths
+
+## Description
+
+The operation `--delete` deletes the store paths paths from the Nix
+store, but only if it is safe to do so; that is, when the path is not
+reachable from a root of the garbage collector. This means that you can
+only delete paths that would also be deleted by `nix-store --gc`. Thus,
+`--delete` is a more targeted version of `--gc`.
+
+With the option `--ignore-liveness`, reachability from the roots is
+ignored. However, the path still won’t be deleted if there are other
+paths in the store that refer to it (i.e., depend on it).
+
+## Example
+
+    $ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4
+    0 bytes freed (0.00 MiB)
+    error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive
+
+# Operation `--query`
+
+## Synopsis
+
+nix-store
+
+\--query
+
+\-q
+
+\--outputs
+
+\--requisites
+
+\-R
+
+\--references
+
+\--referrers
+
+\--referrers-closure
+
+\--deriver
+
+\-d
+
+\--graph
+
+\--tree
+
+\--binding
+
+name
+
+\-b
+
+name
+
+\--hash
+
+\--size
+
+\--roots
+
+\--use-output
+
+\-u
+
+\--force-realise
+
+\-f
+
+paths
+
+## Description
+
+The operation `--query` displays various bits of information about the
+store paths . The queries are described below. At most one query can be
+specified. The default query is `--outputs`.
+
+The paths paths may also be symlinks from outside of the Nix store, to
+the Nix store. In that case, the query is applied to the target of the
+symlink.
+
+## Common query options
+
+  - `--use-output`; `-u`  
+    For each argument to the query that is a store derivation, apply the
+    query to the output path of the derivation instead.
+
+  - `--force-realise`; `-f`  
+    Realise each argument to the query first (see [`nix-store
+                    --realise`](#rsec-nix-store-realise)).
+
+## Queries
+
+  - `--outputs`  
+    Prints out the [output paths](#gloss-output-path) of the store
+    derivations paths. These are the paths that will be produced when
+    the derivation is built.
+
+  - `--requisites`; `-R`  
+    Prints out the [closure](#gloss-closure) of the store path paths.
+    
+    This query has one option:
+    
+      - `--include-outputs`  
+        Also include the output path of store derivations, and their
+        closures.
+    
+    This query can be used to implement various kinds of deployment. A
+    *source deployment* is obtained by distributing the closure of a
+    store derivation. A *binary deployment* is obtained by distributing
+    the closure of an output path. A *cache deployment* (combined
+    source/binary deployment, including binaries of build-time-only
+    dependencies) is obtained by distributing the closure of a store
+    derivation and specifying the option `--include-outputs`.
+
+  - `--references`  
+    Prints the set of [references](#gloss-reference) of the store paths
+    paths, that is, their immediate dependencies. (For *all*
+    dependencies, use `--requisites`.)
+
+  - `--referrers`  
+    Prints the set of *referrers* of the store paths paths, that is, the
+    store paths currently existing in the Nix store that refer to one of
+    paths. Note that contrary to the references, the set of referrers is
+    not constant; it can change as store paths are added or removed.
+
+  - `--referrers-closure`  
+    Prints the closure of the set of store paths paths under the
+    referrers relation; that is, all store paths that directly or
+    indirectly refer to one of paths. These are all the path currently
+    in the Nix store that are dependent on paths.
+
+  - `--deriver`; `-d`  
+    Prints the [deriver](#gloss-deriver) of the store paths paths. If
+    the path has no deriver (e.g., if it is a source file), or if the
+    deriver is not known (e.g., in the case of a binary-only
+    deployment), the string `unknown-deriver` is printed.
+
+  - `--graph`  
+    Prints the references graph of the store paths paths in the format
+    of the `dot` tool of AT\&T's [Graphviz
+    package](http://www.graphviz.org/). This can be used to visualise
+    dependency graphs. To obtain a build-time dependency graph, apply
+    this to a store derivation. To obtain a runtime dependency graph,
+    apply it to an output path.
+
+  - `--tree`  
+    Prints the references graph of the store paths paths as a nested
+    ASCII tree. References are ordered by descending closure size; this
+    tends to flatten the tree, making it more readable. The query only
+    recurses into a store path when it is first encountered; this
+    prevents a blowup of the tree representation of the graph.
+
+  - `--graphml`  
+    Prints the references graph of the store paths paths in the
+    [GraphML](http://graphml.graphdrawing.org/) file format. This can be
+    used to visualise dependency graphs. To obtain a build-time
+    dependency graph, apply this to a store derivation. To obtain a
+    runtime dependency graph, apply it to an output path.
+
+  - `--binding` name; `-b` name  
+    Prints the value of the attribute name (i.e., environment variable)
+    of the store derivations paths. It is an error for a derivation to
+    not have the specified attribute.
+
+  - `--hash`  
+    Prints the SHA-256 hash of the contents of the store paths paths
+    (that is, the hash of the output of `nix-store --dump` on the given
+    paths). Since the hash is stored in the Nix database, this is a fast
+    operation.
+
+  - `--size`  
+    Prints the size in bytes of the contents of the store paths paths —
+    to be precise, the size of the output of `nix-store --dump` on the
+    given paths. Note that the actual disk space required by the store
+    paths may be higher, especially on filesystems with large cluster
+    sizes.
+
+  - `--roots`  
+    Prints the garbage collector roots that point, directly or
+    indirectly, at the store paths paths.
+
+## Examples
+
+Print the closure (runtime dependencies) of the `svn` program in the
+current user environment:
+
+    $ nix-store -qR $(which svn)
+    /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
+    /nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+    ...
+
+Print the build-time dependencies of `svn`:
+
+    $ nix-store -qR $(nix-store -qd $(which svn))
+    /nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
+    /nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
+    /nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
+    ... lots of other paths ...
+
+The difference with the previous example is that we ask the closure of
+the derivation (`-qd`), not the closure of the output path that contains
+`svn`.
+
+Show the build-time dependencies as a tree:
+
+    $ nix-store -q --tree $(nix-store -qd $(which svn))
+    /nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
+    +---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
+    +---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
+    |   +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
+    |   +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
+    ...
+
+Show all paths that depend on the same OpenSSL library as `svn`:
+
+    $ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn)))
+    /nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0
+    /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
+    /nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3
+    /nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5
+
+Show all paths that directly or indirectly depend on the Glibc (C
+library) used by `svn`:
+
+    $ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
+    /nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
+    /nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
+    ...
+
+Note that `ldd` is a command that prints out the dynamic libraries used
+by an ELF executable.
+
+Make a picture of the runtime dependency graph of the current user
+environment:
+
+    $ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps
+    $ gv graph.ps
+
+Show every garbage collector root that points to a store path that
+depends on `svn`:
+
+    $ nix-store -q --roots $(which svn)
+    /nix/var/nix/profiles/default-81-link
+    /nix/var/nix/profiles/default-82-link
+    /nix/var/nix/profiles/per-user/eelco/profile-97-link
+
+# Operation `--add`
+
+## Synopsis
+
+nix-store
+
+\--add
+
+paths
+
+## Description
+
+The operation `--add` adds the specified paths to the Nix store. It
+prints the resulting paths in the Nix store on standard output.
+
+## Example
+
+    $ nix-store --add ./foo.c
+    /nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c
+
+# Operation `--add-fixed`
+
+## Synopsis
+
+nix-store
+
+\--recursive
+
+\--add-fixed
+
+algorithm
+
+paths
+
+## Description
+
+The operation `--add-fixed` adds the specified paths to the Nix store.
+Unlike `--add` paths are registered using the specified hashing
+algorithm, resulting in the same output path as a fixed-output
+derivation. This can be used for sources that are not available from a
+public url or broke since the download expression was written.
+
+This operation has the following options:
+
+  - `--recursive`  
+    Use recursive instead of flat hashing mode, used when adding
+    directories to the store.
+
+## Example
+
+    $ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
+    /nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
+
+# Operation `--verify`
+
+## Synopsis
+
+nix-store
+
+\--verify
+
+\--check-contents
+
+\--repair
+
+## Description
+
+The operation `--verify` verifies the internal consistency of the Nix
+database, and the consistency between the Nix database and the Nix
+store. Any inconsistencies encountered are automatically repaired.
+Inconsistencies are generally the result of the Nix store or database
+being modified by non-Nix tools, or of bugs in Nix itself.
+
+This operation has the following options:
+
+  - `--check-contents`  
+    Checks that the contents of every valid store path has not been
+    altered by computing a SHA-256 hash of the contents and comparing it
+    with the hash stored in the Nix database at build time. Paths that
+    have been modified are printed out. For large stores,
+    `--check-contents` is obviously quite slow.
+
+  - `--repair`  
+    If any valid path is missing from the store, or (if
+    `--check-contents` is given) the contents of a valid path has been
+    modified, then try to repair the path by redownloading it. See
+    `nix-store --repair-path` for details.
+
+# Operation `--verify-path`
+
+## Synopsis
+
+nix-store
+
+\--verify-path
+
+paths
+
+## Description
+
+The operation `--verify-path` compares the contents of the given store
+paths to their cryptographic hashes stored in Nix’s database. For every
+changed path, it prints a warning message. The exit status is 0 if no
+path has changed, and 1 otherwise.
+
+## Example
+
+To verify the integrity of the `svn` command and all its dependencies:
+
+    $ nix-store --verify-path $(nix-store -qR $(which svn))
+
+# Operation `--repair-path`
+
+## Synopsis
+
+nix-store
+
+\--repair-path
+
+paths
+
+## Description
+
+The operation `--repair-path` attempts to “repair” the specified paths
+by redownloading them using the available substituters. If no
+substitutes are available, then repair is not possible.
+
+> **Warning**
+> 
+> During repair, there is a very small time window during which the old
+> path (if it exists) is moved out of the way and replaced with the new
+> path. If repair is interrupted in between, then the system may be left
+> in a broken state (e.g., if the path contains a critical system
+> component like the GNU C Library).
+
+## Example
+
+    $ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
+    path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified!
+      expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588',
+      got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4'
+    
+    $ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
+    fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
+    …
+
+# Operation `--dump`
+
+## Synopsis
+
+nix-store
+
+\--dump
+
+path
+
+## Description
+
+The operation `--dump` produces a NAR (Nix ARchive) file containing the
+contents of the file system tree rooted at path. The archive is written
+to standard output.
+
+A NAR archive is like a TAR or Zip archive, but it contains only the
+information that Nix considers important. For instance, timestamps are
+elided because all files in the Nix store have their timestamp set to 0
+anyway. Likewise, all permissions are left out except for the execute
+bit, because all files in the Nix store have 444 or 555 permission.
+
+Also, a NAR archive is *canonical*, meaning that “equal” paths always
+produce the same NAR archive. For instance, directory entries are always
+sorted so that the actual on-disk order doesn’t influence the result.
+This means that the cryptographic hash of a NAR dump of a path is usable
+as a fingerprint of the contents of the path. Indeed, the hashes of
+store paths stored in Nix’s database (see [`nix-store -q
+--hash`](#refsec-nix-store-query)) are SHA-256 hashes of the NAR dump of
+each store path.
+
+NAR archives support filenames of unlimited length and 64-bit file
+sizes. They can contain regular files, directories, and symbolic links,
+but not other types of files (such as device nodes).
+
+A Nix archive can be unpacked using `nix-store
+--restore`.
+
+# Operation `--restore`
+
+## Synopsis
+
+nix-store
+
+\--restore
+
+path
+
+## Description
+
+The operation `--restore` unpacks a NAR archive to path, which must not
+already exist. The archive is read from standard input.
+
+# Operation `--export`
+
+## Synopsis
+
+nix-store
+
+\--export
+
+paths
+
+## Description
+
+The operation `--export` writes a serialisation of the specified store
+paths to standard output in a format that can be imported into another
+Nix store with `nix-store --import`. This is like `nix-store
+--dump`, except that the NAR archive produced by that command doesn’t
+contain the necessary meta-information to allow it to be imported into
+another Nix store (namely, the set of references of the path).
+
+This command does not produce a *closure* of the specified paths, so if
+a store path references other store paths that are missing in the target
+Nix store, the import will fail. To copy a whole closure, do something
+like:
+
+    $ nix-store --export $(nix-store -qR paths) > out
+
+To import the whole closure again, run:
+
+    $ nix-store --import < out
+
+# Operation `--import`
+
+## Synopsis
+
+nix-store
+
+\--import
+
+## Description
+
+The operation `--import` reads a serialisation of a set of store paths
+produced by `nix-store --export` from standard input and adds those
+store paths to the Nix store. Paths that already exist in the Nix store
+are ignored. If a path refers to another path that doesn’t exist in the
+Nix store, the import fails.
+
+# Operation `--optimise`
+
+## Synopsis
+
+nix-store
+
+\--optimise
+
+## Description
+
+The operation `--optimise` reduces Nix store disk space usage by finding
+identical files in the store and hard-linking them to each other. It
+typically reduces the size of the store by something like 25-35%. Only
+regular files and symlinks are hard-linked in this manner. Files are
+considered identical when they have the same NAR archive serialisation:
+that is, regular files must have the same contents and permission
+(executable or non-executable), and symlinks must have the same
+contents.
+
+After completion, or when the command is interrupted, a report on the
+achieved savings is printed on standard error.
+
+Use `-vv` or `-vvv` to get some progress indication.
+
+## Example
+
+    $ nix-store --optimise
+    hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1'
+    ...
+    541838819 bytes (516.74 MiB) freed by hard-linking 54143 files;
+    there are 114486 files with equal contents out of 215894 files in total
+
+# Operation `--read-log`
+
+## Synopsis
+
+nix-store
+
+\--read-log
+
+\-l
+
+paths
+
+## Description
+
+The operation `--read-log` prints the build log of the specified store
+paths on standard output. The build log is whatever the builder of a
+derivation wrote to standard output and standard error. If a store path
+is not a derivation, the deriver of the store path is used.
+
+Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no
+guarantee that a build log is available for any particular store path.
+For instance, if the path was downloaded as a pre-built binary through a
+substitute, then the log is unavailable.
+
+## Example
+
+    $ nix-store -l $(which ktorrent)
+    building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1
+    unpacking sources
+    unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz
+    ktorrent-2.2.1/
+    ktorrent-2.2.1/NEWS
+    ...
+
+# Operation `--dump-db`
+
+## Synopsis
+
+nix-store
+
+\--dump-db
+
+paths
+
+## Description
+
+The operation `--dump-db` writes a dump of the Nix database to standard
+output. It can be loaded into an empty Nix store using `--load-db`. This
+is useful for making backups and when migrating to different database
+schemas.
+
+By default, `--dump-db` will dump the entire Nix database. When one or
+more store paths is passed, only the subset of the Nix database for
+those store paths is dumped. As with `--export`, the user is responsible
+for passing all the store paths for a closure. See `--export` for an
+example.
+
+# Operation `--load-db`
+
+## Synopsis
+
+nix-store
+
+\--load-db
+
+## Description
+
+The operation `--load-db` reads a dump of the Nix database created by
+`--dump-db` from standard input and loads it into the Nix database.
+
+# Operation `--print-env`
+
+## Synopsis
+
+nix-store
+
+\--print-env
+
+drvpath
+
+## Description
+
+The operation `--print-env` prints out the environment of a derivation
+in a format that can be evaluated by a shell. The command line arguments
+of the builder are placed in the variable `_args`.
+
+## Example
+
+    $ nix-store --print-env $(nix-instantiate '<nixpkgs>' -A firefox)
+    …
+    export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2'
+    export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv'
+    export system; system='x86_64-linux'
+    export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh'
+
+# Operation `--generate-binary-cache-key`
+
+## Synopsis
+
+nix-store
+
+\--generate-binary-cache-key
+
+key-name
+
+secret-key-file
+
+public-key-file
+
+## Description
+
+This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/)
+that can be used to create a signed binary cache. It takes three
+mandatory parameters:
+
+1.  A key name, such as `cache.example.org-1`, that is used to look up
+    keys on the client when it verifies signatures. It can be anything,
+    but it’s suggested to use the host name of your cache (e.g.
+    `cache.example.org`) with a suffix denoting the number of the key
+    (to be incremented every time you need to revoke a key).
+
+2.  The file name where the secret key is to be stored.
+
+3.  The file name where the public key is to be stored.
+
+# Environment variables
diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md
new file mode 100644
index 000000000..b66d318c2
--- /dev/null
+++ b/doc/manual/src/command-ref/opt-common-syn.md
@@ -0,0 +1,57 @@
+\--help
+
+\--version
+
+\--verbose
+
+\-v
+
+\--quiet
+
+\--log-format
+
+format
+
+\--no-build-output
+
+\-Q
+
+\--max-jobs
+
+\-j
+
+number
+
+\--cores
+
+number
+
+\--max-silent-time
+
+number
+
+\--timeout
+
+number
+
+\--keep-going
+
+\-k
+
+\--keep-failed
+
+\-K
+
+\--fallback
+
+\--readonly-mode
+
+\-I
+
+path
+
+\--option
+
+name
+
+value
diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md
new file mode 100644
index 000000000..1703c40e3
--- /dev/null
+++ b/doc/manual/src/command-ref/opt-inst-syn.md
@@ -0,0 +1,15 @@
+\--prebuilt-only
+
+\-b
+
+\--attr
+
+\-A
+
+\--from-expression
+
+\-E
+
+\--from-profile
+
+path

From efff6cf163fb911689b41d7c2970efd8b17876e1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 13:09:30 +0200
Subject: [PATCH 029/882] Install all manpages

---
 doc/manual/local.mk | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 80c517702..13d937f8d 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -12,20 +12,21 @@ dist-files += $(d)/version.txt
 
 # Generate man pages.
 man-pages := $(foreach n, \
-  nix-copy-closure.1 \
-  nix.conf.5, \
+  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
+  nix-collect-garbage.1 \
+  nix-prefetch-url.1 nix-channel.1 \
+  nix-hash.1 nix-copy-closure.1 \
+  nix.conf.5 nix-daemon.8, \
   $(d)/$(n))
-#  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
-#  nix-collect-garbage.1, \
-#  nix-prefetch-url.1 nix-channel.1 \
-#  nix-hash.1 nix-copy-closure.1 \
-#  nix.conf.5 nix-daemon.8, \
 
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 dist-files += $(man-pages)
 
-$(d)/nix-copy-closure.1: $(d)/src/command-ref/nix-copy-closure.md
+$(d)/%.1: $(d)/src/command-ref/%.md
+	$(trace-gen) lowdown -sT man $^ -o $@
+
+$(d)/%.8: $(d)/src/command-ref/%.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md

From 13df1faf25769924dae07e65f3ef3ffdd66f10ee Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 13:58:49 +0200
Subject: [PATCH 030/882] Get rid of callouts since Markdown doesn't support
 them

---
 .../expressions/arguments-variables.xml       | 51 ++++++-------
 doc/manual/expressions/build-script.xml       | 58 ++++++++-------
 doc/manual/expressions/builtins.xml           | 72 ++++++++-----------
 doc/manual/expressions/expression-syntax.xml  | 50 +++++++------
 doc/manual/expressions/generic-builder.xml    | 34 ++++-----
 .../expressions/language-constructs.xml       | 36 +++++-----
 .../src/expressions/arguments-variables.md    | 28 ++++----
 doc/manual/src/expressions/build-script.md    | 32 ++++-----
 doc/manual/src/expressions/builtins.md        | 54 +++++++-------
 .../src/expressions/expression-syntax.md      | 29 ++++----
 doc/manual/src/expressions/generic-builder.md | 18 ++---
 .../src/expressions/language-constructs.md    | 22 +++---
 12 files changed, 233 insertions(+), 251 deletions(-)

diff --git a/doc/manual/expressions/arguments-variables.xml b/doc/manual/expressions/arguments-variables.xml
index bf60cb7ee..a4375c777 100644
--- a/doc/manual/expressions/arguments-variables.xml
+++ b/doc/manual/expressions/arguments-variables.xml
@@ -6,20 +6,25 @@
 
 <title>Arguments and Variables</title>
 
-<example xml:id='ex-hello-composition'>
+<para>The Nix expression in <xref linkend='ex-hello-nix' /> is a
+function; it is missing some arguments that have to be filled in
+somewhere.  In the Nix Packages collection this is done in the file
+<filename>pkgs/top-level/all-packages.nix</filename>, where all Nix
+expressions for packages are imported and called with the appropriate
+arguments. Here are some fragments of
+<filename>all-packages.nix</filename>, with annotations of what they
+mean:</para>
 
-<title>Composing GNU Hello
-(<filename>all-packages.nix</filename>)</title>
 <programlisting>
 ...
 
-rec { <co xml:id='ex-hello-composition-co-1' />
+rec { ①
 
-  hello = import ../applications/misc/hello/ex-1 <co xml:id='ex-hello-composition-co-2' /> { <co xml:id='ex-hello-composition-co-3' />
+  hello = import ../applications/misc/hello/ex-1 ② { ③
     inherit fetchurl stdenv perl;
   };
 
-  perl = import ../development/interpreters/perl { <co xml:id='ex-hello-composition-co-4' />
+  perl = import ../development/interpreters/perl { ④
     inherit fetchurl stdenv;
   };
 
@@ -31,31 +36,19 @@ rec { <co xml:id='ex-hello-composition-co-1' />
 
 }
 </programlisting>
-</example>
 
-<para>The Nix expression in <xref linkend='ex-hello-nix' /> is a
-function; it is missing some arguments that have to be filled in
-somewhere.  In the Nix Packages collection this is done in the file
-<filename>pkgs/top-level/all-packages.nix</filename>, where all
-Nix expressions for packages are imported and called with the
-appropriate arguments.  <xref linkend='ex-hello-composition' /> shows
-some fragments of
-<filename>all-packages.nix</filename>.</para>
-
-<calloutlist>
-
-  <callout arearefs='ex-hello-composition-co-1'>
+<orderedlist>
 
+  <listitem>
     <para>This file defines a set of attributes, all of which are
     concrete derivations (i.e., not functions).  In fact, we define a
     <emphasis>mutually recursive</emphasis> set of attributes.  That
     is, the attributes can refer to each other.  This is precisely
     what we want since we want to <quote>plug</quote> the
     various packages into each other.</para>
+  </listitem>
 
-  </callout>
-
-  <callout arearefs='ex-hello-composition-co-2'>
+  <listitem>
 
     <para>Here we <emphasis>import</emphasis> the Nix expression for
     GNU Hello.  The import operation just loads and returns the
@@ -71,9 +64,9 @@ some fragments of
     When you try to import a directory, Nix automatically appends
     <filename>/default.nix</filename> to the file name.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-composition-co-3'>
+  <listitem>
 
     <para>This is where the actual composition takes place.  Here we
     <emphasis>call</emphasis> the function imported from
@@ -107,15 +100,15 @@ hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; };
 
     </para></note>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-composition-co-4'>
+  <listitem>
 
     <para>Likewise, we have to instantiate Perl,
     <varname>fetchurl</varname>, and the standard environment.</para>
 
-  </callout>
+  </listitem>
 
-</calloutlist>
+</orderedlist>
 
-</section>
\ No newline at end of file
+</section>
diff --git a/doc/manual/expressions/build-script.xml b/doc/manual/expressions/build-script.xml
index 44264239d..892274f50 100644
--- a/doc/manual/expressions/build-script.xml
+++ b/doc/manual/expressions/build-script.xml
@@ -6,32 +6,30 @@
 
 <title>Build Script</title>
 
-<example xml:id='ex-hello-builder'><title>Build script for GNU Hello
-(<filename>builder.sh</filename>)</title>
-<programlisting>
-source $stdenv/setup <co xml:id='ex-hello-builder-co-1' />
-
-PATH=$perl/bin:$PATH <co xml:id='ex-hello-builder-co-2' />
-
-tar xvfz $src <co xml:id='ex-hello-builder-co-3' />
-cd hello-*
-./configure --prefix=$out <co xml:id='ex-hello-builder-co-4' />
-make <co xml:id='ex-hello-builder-co-5' />
-make install</programlisting>
-</example>
-
-<para><xref linkend='ex-hello-builder' /> shows the builder referenced
+<para>Here is the builder referenced
 from Hello's Nix expression (stored in
-<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>).
-The builder can actually be made a lot shorter by using the
+<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>):</para>
+
+<programlisting>
+source $stdenv/setup ①
+
+PATH=$perl/bin:$PATH ②
+
+tar xvfz $src ③
+cd hello-*
+./configure --prefix=$out ④
+make ⑤
+make install</programlisting>
+
+<para>The builder can actually be made a lot shorter by using the
 <emphasis>generic builder</emphasis> functions provided by
 <varname>stdenv</varname>, but here we write out the build steps to
 elucidate what a builder does.  It performs the following
 steps:</para>
 
-<calloutlist>
+<orderedlist>
 
-  <callout arearefs='ex-hello-builder-co-1'>
+  <listitem>
 
     <para>When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the
@@ -52,9 +50,9 @@ steps:</para>
     attribute in <xref linkend='ex-hello-nix' />, but
     <varname>mkDerivation</varname> adds it automatically.)</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-builder-co-2'>
+  <listitem>
 
     <para>Since Hello needs Perl, we have to make sure that Perl is in
     the <literal>PATH</literal>.  The <literal>perl</literal> environment
@@ -63,9 +61,9 @@ steps:</para>
     <filename><replaceable>$perl</replaceable>/bin</filename> is the
     directory containing the Perl interpreter.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-builder-co-3'>
+  <listitem>
 
     <para>Now we have to unpack the sources.  The
     <varname>src</varname> attribute was bound to the result of
@@ -82,9 +80,9 @@ steps:</para>
     always newly created, so you don't have to worry about files from
     previous builds interfering with the current build.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-builder-co-4'>
+  <listitem>
 
     <para>GNU Hello is a typical Autoconf-based package, so we first
     have to run its <filename>configure</filename> script.  In Nix
@@ -98,17 +96,17 @@ steps:</para>
     <literal>--prefix=$out</literal> to cause Hello to be installed in
     the expected location.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-builder-co-5'>
+  <listitem>
 
     <para>Finally we build Hello (<literal>make</literal>) and install
     it into the location specified by <literal>out</literal>
     (<literal>make install</literal>).</para>
 
-  </callout>
+  </listitem>
 
-</calloutlist>
+</orderedlist>
 
 <para>If you are wondering about the absence of error checking on the
 result of various commands called in the builder: this is because the
@@ -116,4 +114,4 @@ shell script is evaluated with Bash's <option>-e</option> option,
 which causes the script to be aborted if any command fails without an
 error check.</para>
 
-</section>
\ No newline at end of file
+</section>
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index a18c5801a..478b67ce8 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1511,39 +1511,9 @@ in foo</programlisting>
     <!-- TODO: more formally describe the schema of the XML
     representation -->
 
-    <para><xref linkend='ex-toxml' /> shows an example where this is
-    the case.  The builder is supposed to generate the configuration
-    file for a <link xlink:href='http://jetty.mortbay.org/'>Jetty
-    servlet container</link>.  A servlet container contains a number
-    of servlets (<filename>*.war</filename> files) each exported under
-    a specific URI prefix.  So the servlet configuration is a list of
-    sets containing the <varname>path</varname> and
-    <varname>war</varname> of the servlet (<xref
-    linkend='ex-toxml-co-servlets' />).  This kind of information is
-    difficult to communicate with the normal method of passing
-    information through an environment variable, which just
-    concatenates everything together into a string (which might just
-    work in this case, but wouldn’t work if fields are optional or
-    contain lists themselves).  Instead the Nix expression is
-    converted to an XML representation with
-    <function>toXML</function>, which is unambiguous and can easily be
-    processed with the appropriate tools.  For instance, in the
-    example an XSLT stylesheet (<xref linkend='ex-toxml-co-stylesheet'
-    />) is applied to it (<xref linkend='ex-toxml-co-apply' />) to
-    generate the XML configuration file for the Jetty server.  The XML
-    representation produced from <xref linkend='ex-toxml-co-servlets'
-    /> by <function>toXML</function> is shown in <xref
-    linkend='ex-toxml-result' />.</para>
-
-    <para>Note that <xref linkend='ex-toxml' /> uses the <function
-    linkend='builtin-toFile'>toFile</function> built-in to write the
-    builder and the stylesheet “inline” in the Nix expression.  The
-    path of the stylesheet is spliced into the builder at
-    <literal>xsltproc ${stylesheet}
-    <replaceable>...</replaceable></literal>.</para>
-
-    <example xml:id='ex-toxml'><title>Passing information to a builder
-    using <function>toXML</function></title>
+    <para>Here is an example where this is
+    the case:
+    </para>
 
 <programlisting><![CDATA[
 { stdenv, fetchurl, libxslt, jira, uberwiki }:
@@ -1556,10 +1526,10 @@ stdenv.mkDerivation (rec {
   builder = builtins.toFile "builder.sh" "
     source $stdenv/setup
     mkdir $out
-    echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[
+    echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> ① <![CDATA[
   ";
 
-  stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[
+  stylesheet = builtins.toFile "stylesheet.xsl"]]> ② <![CDATA[
    "<?xml version='1.0' encoding='UTF-8'?>
     <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
       <xsl:template match='/'>
@@ -1575,16 +1545,32 @@ stdenv.mkDerivation (rec {
     </xsl:stylesheet>
   ";
 
-  servlets = builtins.toXML []]> <co xml:id='ex-toxml-co-servlets' /> <![CDATA[
+  servlets = builtins.toXML []]> ③ <![CDATA[
     { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
     { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
   ];
 })]]></programlisting>
 
-    </example>
-
-    <example xml:id='ex-toxml-result'><title>XML representation produced by
-    <function>toXML</function></title>
+    <para>The builder is supposed to generate the configuration file
+    for a <link xlink:href='http://jetty.mortbay.org/'>Jetty servlet
+    container</link>.  A servlet container contains a number of
+    servlets (<filename>*.war</filename> files) each exported under a
+    specific URI prefix.  So the servlet configuration is a list of
+    sets containing the <varname>path</varname> and
+    <varname>war</varname> of the servlet (<xref
+    linkend='ex-toxml-co-servlets' />).  This kind of information is
+    difficult to communicate with the normal method of passing
+    information through an environment variable, which just
+    concatenates everything together into a string (which might just
+    work in this case, but wouldn’t work if fields are optional or
+    contain lists themselves).  Instead the Nix expression is
+    converted to an XML representation with
+    <function>toXML</function>, which is unambiguous and can easily be
+    processed with the appropriate tools.  For instance, in the
+    example an XSLT stylesheet (at point ②) is applied to it (at point
+    ①) to generate the XML configuration file for the Jetty server.
+    The XML representation produced at point ③ by
+    <function>toXML</function> is as follows:</para>
 
 <programlisting><![CDATA[<?xml version='1.0' encoding='utf-8'?>
 <expr>
@@ -1608,7 +1594,11 @@ stdenv.mkDerivation (rec {
   </list>
 </expr>]]></programlisting>
 
-    </example>
+    <para>Note that <xref linkend='ex-toxml' /> uses the <function
+    linkend='builtin-toFile'>toFile</function> built-in to write the
+    builder and the stylesheet “inline” in the Nix expression.  The
+    path of the stylesheet is spliced into the builder using the
+    syntax <literal>xsltproc ${stylesheet}</literal>.</para>
 
     </listitem>
 
diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml
index a3de20713..562a9e9f4 100644
--- a/doc/manual/expressions/expression-syntax.xml
+++ b/doc/manual/expressions/expression-syntax.xml
@@ -6,33 +6,31 @@
 
 <title>Expression Syntax</title>
 
-<example xml:id='ex-hello-nix'><title>Nix expression for GNU Hello
-(<filename>default.nix</filename>)</title>
-<programlisting>
-{ stdenv, fetchurl, perl }: <co xml:id='ex-hello-nix-co-1' />
+<para>Here is a Nix expression for GNU Hello:</para>
 
-stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
-  name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
-  builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
-  src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
+<programlisting>
+{ stdenv, fetchurl, perl }: ①
+
+stdenv.mkDerivation { ②
+  name = "hello-2.1.1"; ③
+  builder = ./builder.sh; ④
+  src = fetchurl { ⑤
     url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
-  inherit perl; <co xml:id='ex-hello-nix-co-6' />
+  inherit perl; ⑥
 }</programlisting>
-</example>
 
-<para><xref linkend='ex-hello-nix' /> shows a Nix expression for GNU
-Hello.  It's actually already in the Nix Packages collection in
+<para>This file is actually already in the Nix Packages collection in
 <filename>pkgs/applications/misc/hello/ex-1/default.nix</filename>.
 It is customary to place each package in a separate directory and call
 the single Nix expression in that directory
 <filename>default.nix</filename>.  The file has the following elements
 (referenced from the figure by number):
 
-<calloutlist>
+<orderedlist>
 
-  <callout arearefs='ex-hello-nix-co-1'>
+  <listitem>
 
     <para>This states that the expression is a
     <emphasis>function</emphasis> that expects to be called with three
@@ -57,9 +55,9 @@ the single Nix expression in that directory
     function; when given the required arguments, the body should
     describe how to build an instance of the Hello package.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-nix-co-2'>
+  <listitem>
 
     <para>So we have to build a package.  Building something from
     other stuff is called a <emphasis>derivation</emphasis> in Nix (as
@@ -76,9 +74,9 @@ the single Nix expression in that directory
     <replaceable>nameN</replaceable> =
     <replaceable>exprN</replaceable>; }</literal>.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-nix-co-3'>
+  <listitem>
 
     <para>The attribute <varname>name</varname> specifies the symbolic
     name and version of the package.  Nix doesn't really care about
@@ -87,9 +85,9 @@ the single Nix expression in that directory
     packages.  This attribute is required by
     <varname>mkDerivation</varname>.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-nix-co-4'>
+  <listitem>
 
     <para>The attribute <varname>builder</varname> specifies the
     builder.  This attribute can sometimes be omitted, in which case
@@ -101,9 +99,9 @@ the single Nix expression in that directory
     <command>./builder.sh</command> refers to the shell script shown
     in <xref linkend='ex-hello-builder' />, discussed below.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-nix-co-5'>
+  <listitem>
 
     <para>The builder has to know what the sources of the package
     are.  Here, the attribute <varname>src</varname> is bound to the
@@ -120,9 +118,9 @@ the single Nix expression in that directory
     customary, and it's also expected by the default builder (which we
     don't use in this example).</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-nix-co-6'>
+  <listitem>
 
     <para>Since the derivation requires Perl, we have to pass the
     value of the <varname>perl</varname> function argument to the
@@ -139,9 +137,9 @@ perl = perl;</programlisting>
     causes the specified attributes to be bound to whatever variables
     with the same name happen to be in scope.</para>
 
-  </callout>
+  </listitem>
 
-</calloutlist>
+</orderedlist>
 
 </para>
 
diff --git a/doc/manual/expressions/generic-builder.xml b/doc/manual/expressions/generic-builder.xml
index 16b0268a7..71b342f99 100644
--- a/doc/manual/expressions/generic-builder.xml
+++ b/doc/manual/expressions/generic-builder.xml
@@ -20,23 +20,21 @@ make install</programlisting>
 The builders for almost all Unix packages look like this — set up some
 environment variables, unpack the sources, configure, build, and
 install.  For this reason the standard environment provides some Bash
-functions that automate the build process.  A builder using the
-generic build facilities in shown in <xref linkend='ex-hello-builder2'
-/>.</para>
+functions that automate the build process.  Here is what a builder
+using the generic build facilities looks like:</para>
 
-<example xml:id='ex-hello-builder2'><title>Build script using the generic
-build functions</title>
 <programlisting>
-buildInputs="$perl" <co xml:id='ex-hello-builder2-co-1' />
+buildInputs="$perl" ①
 
-source $stdenv/setup <co xml:id='ex-hello-builder2-co-2' />
+source $stdenv/setup ②
 
-genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting>
-</example>
+genericBuild ③</programlisting>
 
-<calloutlist>
+<para>Here is what each line means:
 
-  <callout arearefs='ex-hello-builder2-co-1'>
+<orderedlist>
+
+  <listitem>
 
     <para>The <literal>buildInputs</literal> variable tells
     <filename>setup</filename> to use the indicated packages as
@@ -54,16 +52,16 @@ genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting>
     inputs.</para></footnote>
     </para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-builder2-co-2'>
+  <listitem arearefs='ex-hello-builder2-co-2'>
 
     <para>The function <function>genericBuild</function> is defined in
     the file <literal>$stdenv/setup</literal>.</para>
 
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-hello-builder2-co-3'>
+  <listitem arearefs='ex-hello-builder2-co-3'>
 
     <para>The final step calls the shell function
     <function>genericBuild</function>, which performs the steps that
@@ -73,9 +71,11 @@ genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting>
     <command>bzip2</command>, etc.  It can be customised in many ways;
     see the Nixpkgs manual for details.</para>
 
-  </callout>
+  </listitem>
 
-</calloutlist>
+</orderedlist>
+
+</para>
 
 <para>Discerning readers will note that the
 <literal>buildInputs</literal> could just as well have been set in the Nix
diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 0d0cbbe15..4d316609c 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -278,7 +278,9 @@ evaluate to a Boolean value.  If it evaluates to
 <literal>true</literal>, <replaceable>e2</replaceable> is returned;
 otherwise expression evaluation is aborted and a backtrace is printed.</para>
 
-<example xml:id='ex-subversion-nix'><title>Nix expression for Subversion</title>
+<para>Here is a Nix expression for the Subversion package that shows
+how assertions can be used:.</para>
+
 <programlisting>
 { localServer ? false
 , httpServer ? false
@@ -290,9 +292,9 @@ otherwise expression evaluation is aborted and a backtrace is printed.</para>
 , openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null
 }:
 
-assert localServer -> db4 != null; <co xml:id='ex-subversion-nix-co-1' />
-assert httpServer -> httpd != null &amp;&amp; httpd.expat == expat; <co xml:id='ex-subversion-nix-co-2' />
-assert sslSupport -> openssl != null &amp;&amp; (httpServer -> httpd.openssl == openssl); <co xml:id='ex-subversion-nix-co-3' />
+assert localServer -> db4 != null; ①
+assert httpServer -> httpd != null &amp;&amp; httpd.expat == expat; ②
+assert sslSupport -> openssl != null &amp;&amp; (httpServer -> httpd.openssl == openssl); ③
 assert pythonBindings -> swig != null &amp;&amp; swig.pythonSupport;
 assert javaSwigBindings -> swig != null &amp;&amp; swig.javaSupport;
 assert javahlBindings -> j2sdk != null;
@@ -300,26 +302,24 @@ assert javahlBindings -> j2sdk != null;
 stdenv.mkDerivation {
   name = "subversion-1.1.1";
   ...
-  openssl = if sslSupport then openssl else null; <co xml:id='ex-subversion-nix-co-4' />
+  openssl = if sslSupport then openssl else null; ④
   ...
 }</programlisting>
-</example>
 
-<para><xref linkend='ex-subversion-nix' /> show how assertions are
-used in the Nix expression for Subversion.</para>
+<para>The points of interest are:</para>
 
-<calloutlist>
+<orderedlist>
 
-  <callout arearefs='ex-subversion-nix-co-1'>
+  <listitem>
     <para>This assertion states that if Subversion is to have support
     for local repositories, then Berkeley DB is needed.  So if the
     Subversion function is called with the
     <varname>localServer</varname> argument set to
     <literal>true</literal> but the <varname>db4</varname> argument
     set to <literal>null</literal>, then the evaluation fails.</para>
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-subversion-nix-co-2'>
+  <listitem>
     <para>This is a more subtle condition: if Subversion is built with
     Apache (<literal>httpServer</literal>) support, then the Expat
     library (an XML library) used by Subversion should be same as the
@@ -327,27 +327,27 @@ used in the Nix expression for Subversion.</para>
     Subversion code ends up being linked with Apache code, and if the
     Expat libraries do not match, a build- or runtime link error or
     incompatibility might occur.</para>
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-subversion-nix-co-3'>
+  <listitem>
     <para>This assertion says that in order for Subversion to have SSL
     support (so that it can access <literal>https</literal> URLs), an
     OpenSSL library must be passed.  Additionally, it says that
     <emphasis>if</emphasis> Apache support is enabled, then Apache's
     OpenSSL should match Subversion's.  (Note that if Apache support
     is not enabled, we don't care about Apache's OpenSSL.)</para>
-  </callout>
+  </listitem>
 
-  <callout arearefs='ex-subversion-nix-co-4'>
+  <listitem>
     <para>The conditional here is not really related to assertions,
     but is worth pointing out: it ensures that if SSL support is
     disabled, then the Subversion derivation is not dependent on
     OpenSSL, even if a non-<literal>null</literal> value was passed.
     This prevents an unnecessary rebuild of Subversion if OpenSSL
     changes.</para>
-  </callout>
+  </listitem>
 
-</calloutlist>
+</orderedlist>
 
 </simplesect>
 
diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md
index 9a373d94d..436ae559d 100644
--- a/doc/manual/src/expressions/arguments-variables.md
+++ b/doc/manual/src/expressions/arguments-variables.md
@@ -1,14 +1,21 @@
 # Arguments and Variables
 
+The Nix expression in [???](#ex-hello-nix) is a function; it is missing
+some arguments that have to be filled in somewhere. In the Nix Packages
+collection this is done in the file `pkgs/top-level/all-packages.nix`,
+where all Nix expressions for packages are imported and called with the
+appropriate arguments. Here are some fragments of `all-packages.nix`,
+with annotations of what they mean:
+
     ...
     
-    rec { 
+    rec { ①
     
-      hello = import ../applications/misc/hello/ex-1  { 
+      hello = import ../applications/misc/hello/ex-1 ② { ③
         inherit fetchurl stdenv perl;
       };
     
-      perl = import ../development/interpreters/perl { 
+      perl = import ../development/interpreters/perl { ④
         inherit fetchurl stdenv;
       };
     
@@ -20,20 +27,13 @@
     
     }
 
-The Nix expression in [???](#ex-hello-nix) is a function; it is missing
-some arguments that have to be filled in somewhere. In the Nix Packages
-collection this is done in the file `pkgs/top-level/all-packages.nix`,
-where all Nix expressions for packages are imported and called with the
-appropriate arguments. [example\_title](#ex-hello-composition) shows
-some fragments of `all-packages.nix`.
-
-  - This file defines a set of attributes, all of which are concrete
+1.  This file defines a set of attributes, all of which are concrete
     derivations (i.e., not functions). In fact, we define a *mutually
     recursive* set of attributes. That is, the attributes can refer to
     each other. This is precisely what we want since we want to “plug”
     the various packages into each other.
 
-  - Here we *import* the Nix expression for GNU Hello. The import
+2.  Here we *import* the Nix expression for GNU Hello. The import
     operation just loads and returns the specified Nix expression. In
     fact, we could just have put the contents of [???](#ex-hello-nix) in
     `all-packages.nix` at this point. That would be completely
@@ -44,7 +44,7 @@ some fragments of `all-packages.nix`.
     import a directory, Nix automatically appends `/default.nix` to the
     file name.
 
-  - This is where the actual composition takes place. Here we *call* the
+3.  This is where the actual composition takes place. Here we *call* the
     function imported from `../applications/misc/hello/ex-1` with a set
     containing the things that the function expects, namely `fetchurl`,
     `stdenv`, and `perl`. We use inherit again to use the attributes
@@ -68,5 +68,5 @@ some fragments of `all-packages.nix`.
     > 
     >     hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; };
 
-  - Likewise, we have to instantiate Perl, `fetchurl`, and the standard
+4.  Likewise, we have to instantiate Perl, `fetchurl`, and the standard
     environment.
diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md
index a359ebc46..f922182c2 100644
--- a/doc/manual/src/expressions/build-script.md
+++ b/doc/manual/src/expressions/build-script.md
@@ -1,23 +1,23 @@
 # Build Script
 
-    source $stdenv/setup 
+Here is the builder referenced from Hello's Nix expression (stored in
+`pkgs/applications/misc/hello/ex-1/builder.sh`):
+
+    source $stdenv/setup ①
     
-    PATH=$perl/bin:$PATH 
+    PATH=$perl/bin:$PATH ②
     
-    tar xvfz $src 
+    tar xvfz $src ③
     cd hello-*
-    ./configure --prefix=$out 
-    make 
+    ./configure --prefix=$out ④
+    make ⑤
     make install
 
-[example\_title](#ex-hello-builder) shows the builder referenced from
-Hello's Nix expression (stored in
-`pkgs/applications/misc/hello/ex-1/builder.sh`). The builder can
-actually be made a lot shorter by using the *generic builder* functions
-provided by `stdenv`, but here we write out the build steps to elucidate
-what a builder does. It performs the following steps:
+The builder can actually be made a lot shorter by using the *generic
+builder* functions provided by `stdenv`, but here we write out the build
+steps to elucidate what a builder does. It performs the following steps:
 
-  - When Nix runs a builder, it initially completely clears the
+1.  When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the derivation).
     For instance, the `PATH` variable is empty\[1\]. This is done to
     prevent undeclared inputs from being used in the build process. If
@@ -31,13 +31,13 @@ what a builder does. It performs the following steps:
     attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
     automatically.)
 
-  - Since Hello needs Perl, we have to make sure that Perl is in the
+2.  Since Hello needs Perl, we have to make sure that Perl is in the
     `PATH`. The `perl` environment variable points to the location of
     the Perl package (since it was passed in as an attribute to the
     derivation), so `$perl/bin` is the directory containing the Perl
     interpreter.
 
-  - Now we have to unpack the sources. The `src` attribute was bound to
+3.  Now we have to unpack the sources. The `src` attribute was bound to
     the result of fetching the Hello source tarball from the network, so
     the `src` environment variable points to the location in the Nix
     store to which the tarball was downloaded. After unpacking, we `cd`
@@ -50,7 +50,7 @@ what a builder does. It performs the following steps:
     have to worry about files from previous builds interfering with the
     current build.
 
-  - GNU Hello is a typical Autoconf-based package, so we first have to
+4.  GNU Hello is a typical Autoconf-based package, so we first have to
     run its `configure` script. In Nix every package is stored in a
     separate location in the Nix store, for instance
     `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix
@@ -60,7 +60,7 @@ what a builder does. It performs the following steps:
     `--prefix=$out` to cause Hello to be installed in the expected
     location.
 
-  - Finally we build Hello (`make`) and install it into the location
+5.  Finally we build Hello (`make`) and install it into the location
     specified by `out` (`make install`).
 
 If you are wondering about the absence of error checking on the result
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index a378fe8e4..f60af9d24 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -735,31 +735,7 @@ For instance, `derivation` is also available as `builtins.derivation`.
     builder in a more structured format than plain environment
     variables.
     
-    [example\_title](#ex-toxml) shows an example where this is the case.
-    The builder is supposed to generate the configuration file for a
-    [Jetty servlet container](http://jetty.mortbay.org/). A servlet
-    container contains a number of servlets (`*.war` files) each
-    exported under a specific URI prefix. So the servlet configuration
-    is a list of sets containing the `path` and `war` of the servlet
-    ([co\_title](#ex-toxml-co-servlets)). This kind of information is
-    difficult to communicate with the normal method of passing
-    information through an environment variable, which just concatenates
-    everything together into a string (which might just work in this
-    case, but wouldn’t work if fields are optional or contain lists
-    themselves). Instead the Nix expression is converted to an XML
-    representation with `toXML`, which is unambiguous and can easily be
-    processed with the appropriate tools. For instance, in the example
-    an XSLT stylesheet ([co\_title](#ex-toxml-co-stylesheet)) is applied
-    to it ([co\_title](#ex-toxml-co-apply)) to generate the XML
-    configuration file for the Jetty server. The XML representation
-    produced from [co\_title](#ex-toxml-co-servlets) by `toXML` is shown
-    in [example\_title](#ex-toxml-result).
-    
-    Note that [example\_title](#ex-toxml) uses the `toFile` built-in to
-    write the builder and the stylesheet “inline” in the Nix expression.
-    The path of the stylesheet is spliced into the builder at `xsltproc
-    ${stylesheet}
-                    ...`.
+    Here is an example where this is the case:
     
         { stdenv, fetchurl, libxslt, jira, uberwiki }:
         
@@ -771,10 +747,10 @@ For instance, `derivation` is also available as `builtins.derivation`.
           builder = builtins.toFile "builder.sh" "
             source $stdenv/setup
             mkdir $out
-            echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml  
+            echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ① 
           ";
         
-          stylesheet = builtins.toFile "stylesheet.xsl"  
+          stylesheet = builtins.toFile "stylesheet.xsl" ② 
            "<?xml version='1.0' encoding='UTF-8'?>
             <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
               <xsl:template match='/'>
@@ -790,12 +766,29 @@ For instance, `derivation` is also available as `builtins.derivation`.
             </xsl:stylesheet>
           ";
         
-          servlets = builtins.toXML [  
+          servlets = builtins.toXML [ ③ 
             { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
             { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
           ];
         })
     
+    The builder is supposed to generate the configuration file for a
+    [Jetty servlet container](http://jetty.mortbay.org/). A servlet
+    container contains a number of servlets (`*.war` files) each
+    exported under a specific URI prefix. So the servlet configuration
+    is a list of sets containing the `path` and `war` of the servlet
+    ([???](#ex-toxml-co-servlets)). This kind of information is
+    difficult to communicate with the normal method of passing
+    information through an environment variable, which just concatenates
+    everything together into a string (which might just work in this
+    case, but wouldn’t work if fields are optional or contain lists
+    themselves). Instead the Nix expression is converted to an XML
+    representation with `toXML`, which is unambiguous and can easily be
+    processed with the appropriate tools. For instance, in the example
+    an XSLT stylesheet (at point ②) is applied to it (at point ①) to
+    generate the XML configuration file for the Jetty server. The XML
+    representation produced at point ③ by `toXML` is as follows:
+    
         <?xml version='1.0' encoding='utf-8'?>
         <expr>
           <list>
@@ -817,6 +810,11 @@ For instance, `derivation` is also available as `builtins.derivation`.
             </attrs>
           </list>
         </expr>
+    
+    Note that [???](#ex-toxml) uses the `toFile` built-in to write the
+    builder and the stylesheet “inline” in the Nix expression. The path
+    of the stylesheet is spliced into the builder using the syntax
+    `xsltproc ${stylesheet}`.
 
   - `builtins.trace` e1 e2  
     Evaluate e1 and print its abstract syntax representation on standard
diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md
index 935484c33..9e99a1f60 100644
--- a/doc/manual/src/expressions/expression-syntax.md
+++ b/doc/manual/src/expressions/expression-syntax.md
@@ -1,25 +1,26 @@
 # Expression Syntax
 
-    { stdenv, fetchurl, perl }: 
+Here is a Nix expression for GNU Hello:
+
+    { stdenv, fetchurl, perl }: ①
     
-    stdenv.mkDerivation { 
-      name = "hello-2.1.1"; 
-      builder = ./builder.sh; 
-      src = fetchurl { 
+    stdenv.mkDerivation { ②
+      name = "hello-2.1.1"; ③
+      builder = ./builder.sh; ④
+      src = fetchurl { ⑤
         url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
         sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
       };
-      inherit perl; 
+      inherit perl; ⑥
     }
 
-[example\_title](#ex-hello-nix) shows a Nix expression for GNU Hello.
-It's actually already in the Nix Packages collection in
+This file is actually already in the Nix Packages collection in
 `pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to
 place each package in a separate directory and call the single Nix
 expression in that directory `default.nix`. The file has the following
 elements (referenced from the figure by number):
 
-  - This states that the expression is a *function* that expects to be
+1.  This states that the expression is a *function* that expects to be
     called with three arguments: `stdenv`, `fetchurl`, and `perl`. They
     are needed to build Hello, but we don't know how to build them here;
     that's why they are function arguments. `stdenv` is a package that
@@ -37,7 +38,7 @@ elements (referenced from the figure by number):
     the required arguments, the body should describe how to build an
     instance of the Hello package.
 
-  - So we have to build a package. Building something from other stuff
+2.  So we have to build a package. Building something from other stuff
     is called a *derivation* in Nix (as opposed to sources, which are
     built by humans instead of computers). We perform a derivation by
     calling `stdenv.mkDerivation`. `mkDerivation` is a function provided
@@ -50,13 +51,13 @@ elements (referenced from the figure by number):
                     nameN =
                     exprN; }`.
 
-  - The attribute `name` specifies the symbolic name and version of the
+3.  The attribute `name` specifies the symbolic name and version of the
     package. Nix doesn't really care about these things, but they are
     used by for instance `nix-env
                     -q` to show a “human-readable” name for packages. This attribute is
     required by `mkDerivation`.
 
-  - The attribute `builder` specifies the builder. This attribute can
+4.  The attribute `builder` specifies the builder. This attribute can
     sometimes be omitted, in which case `mkDerivation` will fill in a
     default builder (which does a `configure; make; make install`, in
     essence). Hello is sufficiently simple that the default builder
@@ -64,7 +65,7 @@ elements (referenced from the figure by number):
     educational purposes. The value `./builder.sh` refers to the shell
     script shown in [???](#ex-hello-builder), discussed below.
 
-  - The builder has to know what the sources of the package are. Here,
+5.  The builder has to know what the sources of the package are. Here,
     the attribute `src` is bound to the result of a call to the
     `fetchurl` function. Given a URL and a SHA-256 hash of the expected
     contents of the file at that URL, this function builds a derivation
@@ -77,7 +78,7 @@ elements (referenced from the figure by number):
     However, `src` is customary, and it's also expected by the default
     builder (which we don't use in this example).
 
-  - Since the derivation requires Perl, we have to pass the value of the
+6.  Since the derivation requires Perl, we have to pass the value of the
     `perl` function argument to the builder. All attributes in the set
     are actually passed as environment variables to the builder, so
     declaring an attribute
diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md
index a00b08b55..90bdc556b 100644
--- a/doc/manual/src/expressions/generic-builder.md
+++ b/doc/manual/src/expressions/generic-builder.md
@@ -13,24 +13,26 @@ like this:
 The builders for almost all Unix packages look like this — set up some
 environment variables, unpack the sources, configure, build, and
 install. For this reason the standard environment provides some Bash
-functions that automate the build process. A builder using the generic
-build facilities in shown in [example\_title](#ex-hello-builder2).
+functions that automate the build process. Here is what a builder using
+the generic build facilities looks like:
 
-    buildInputs="$perl" 
+    buildInputs="$perl" ①
     
-    source $stdenv/setup 
+    source $stdenv/setup ②
     
-    genericBuild 
+    genericBuild ③
 
-  - The `buildInputs` variable tells `setup` to use the indicated
+Here is what each line means:
+
+1.  The `buildInputs` variable tells `setup` to use the indicated
     packages as “inputs”. This means that if a package provides a `bin`
     subdirectory, it's added to `PATH`; if it has a `include`
     subdirectory, it's added to GCC's header search path; and so
     on.\[1\]
 
-  - The function `genericBuild` is defined in the file `$stdenv/setup`.
+2.  The function `genericBuild` is defined in the file `$stdenv/setup`.
 
-  - The final step calls the shell function `genericBuild`, which
+3.  The final step calls the shell function `genericBuild`, which
     performs the steps that were done explicitly in
     [???](#ex-hello-builder). The generic builder is smart enough to
     figure out whether to unpack the sources using `gzip`, `bzip2`, etc.
diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/expressions/language-constructs.md
index a773c239b..121e4d998 100644
--- a/doc/manual/src/expressions/language-constructs.md
+++ b/doc/manual/src/expressions/language-constructs.md
@@ -203,6 +203,9 @@ where e1 is an expression that should evaluate to a Boolean value. If it
 evaluates to `true`, e2 is returned; otherwise expression evaluation is
 aborted and a backtrace is printed.
 
+Here is a Nix expression for the Subversion package that shows how
+assertions can be used:.
+
     { localServer ? false
     , httpServer ? false
     , sslSupport ? false
@@ -213,9 +216,9 @@ aborted and a backtrace is printed.
     , openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null
     }:
     
-    assert localServer -> db4 != null; 
-    assert httpServer -> httpd != null && httpd.expat == expat; 
-    assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); 
+    assert localServer -> db4 != null; ①
+    assert httpServer -> httpd != null && httpd.expat == expat; ②
+    assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③
     assert pythonBindings -> swig != null && swig.pythonSupport;
     assert javaSwigBindings -> swig != null && swig.javaSupport;
     assert javahlBindings -> j2sdk != null;
@@ -223,33 +226,32 @@ aborted and a backtrace is printed.
     stdenv.mkDerivation {
       name = "subversion-1.1.1";
       ...
-      openssl = if sslSupport then openssl else null; 
+      openssl = if sslSupport then openssl else null; ④
       ...
     }
 
-[example\_title](#ex-subversion-nix) show how assertions are used in the
-Nix expression for Subversion.
+The points of interest are:
 
-  - This assertion states that if Subversion is to have support for
+1.  This assertion states that if Subversion is to have support for
     local repositories, then Berkeley DB is needed. So if the Subversion
     function is called with the `localServer` argument set to `true` but
     the `db4` argument set to `null`, then the evaluation fails.
 
-  - This is a more subtle condition: if Subversion is built with Apache
+2.  This is a more subtle condition: if Subversion is built with Apache
     (`httpServer`) support, then the Expat library (an XML library) used
     by Subversion should be same as the one used by Apache. This is
     because in this configuration Subversion code ends up being linked
     with Apache code, and if the Expat libraries do not match, a build-
     or runtime link error or incompatibility might occur.
 
-  - This assertion says that in order for Subversion to have SSL support
+3.  This assertion says that in order for Subversion to have SSL support
     (so that it can access `https` URLs), an OpenSSL library must be
     passed. Additionally, it says that *if* Apache support is enabled,
     then Apache's OpenSSL should match Subversion's. (Note that if
     Apache support is not enabled, we don't care about Apache's
     OpenSSL.)
 
-  - The conditional here is not really related to assertions, but is
+4.  The conditional here is not really related to assertions, but is
     worth pointing out: it ensures that if SSL support is disabled, then
     the Subversion derivation is not dependent on OpenSSL, even if a
     non-`null` value was passed. This prevents an unnecessary rebuild of

From ca130b73a02f0fbe1d7ef2007d4cd82565eb5eff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 14:10:59 +0200
Subject: [PATCH 031/882] Get rid of <example>

Markdown doesn't have floats so we can't have this.
---
 doc/manual/expressions/builtins.xml           | 117 +++++++++--------
 doc/manual/packages/copy-closure.xml          |   2 +-
 doc/manual/packages/s3-substituter.xml        |  29 +++--
 doc/manual/src/expressions/builtins.md        | 122 +++++++++---------
 .../src/package-management/copy-closure.md    |   2 +-
 .../src/package-management/s3-substituter.md  |  18 +--
 6 files changed, 156 insertions(+), 134 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 478b67ce8..877fd3a1b 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -434,88 +434,93 @@ stdenv.mkDerivation { … }
         </varlistentry>
       </variablelist>
 
-      <example>
-        <title>Fetching a private repository over SSH</title>
-        <programlisting>builtins.fetchGit {
+      <para>Here are some examples of how to use
+      <literal>fetchGit</literal>.</para>
+
+      <itemizedlist>
+
+        <listitem>
+          <para>To fetch a private repository over SSH:</para>
+
+          <programlisting>builtins.fetchGit {
   url = "git@github.com:my-secret/repository.git";
   ref = "master";
   rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
 }</programlisting>
-      </example>
 
-      <example>
-        <title>Fetching an arbitrary ref</title>
-        <programlisting>builtins.fetchGit {
+        </listitem>
+
+        <listitem>
+          <para>To fetch an arbitrary reference:</para>
+
+          <programlisting>builtins.fetchGit {
   url = "https://github.com/NixOS/nix.git";
   ref = "refs/heads/0.5-release";
 }</programlisting>
-      </example>
+        </listitem>
 
-      <example>
-        <title>Fetching a repository's specific commit on an arbitrary branch</title>
-        <para>
-          If the revision you're looking for is in the default branch
-          of the git repository you don't strictly need to specify
-          the branch name in the <varname>ref</varname> attribute.
-        </para>
-        <para>
-          However, if the revision you're looking for is in a future
-          branch for the non-default branch you will need to specify
-          the the <varname>ref</varname> attribute as well.
-        </para>
-        <programlisting>builtins.fetchGit {
+        <listitem>
+          <para>
+            If the revision you're looking for is in the default branch
+            of the git repository you don't strictly need to specify
+            the branch name in the <varname>ref</varname> attribute.
+          </para>
+          <para>
+            However, if the revision you're looking for is in a future
+            branch for the non-default branch you will need to specify
+            the the <varname>ref</varname> attribute as well.
+          </para>
+
+          <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
   rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
   ref = "1.11-maintenance";
 }</programlisting>
-        <note>
-          <para>
-            It is nice to always specify the branch which a revision
-            belongs to. Without the branch being specified, the
-            fetcher might fail if the default branch changes.
-            Additionally, it can be confusing to try a commit from a
-            non-default branch and see the fetch fail. If the branch
-            is specified the fault is much more obvious.
-          </para>
-        </note>
-      </example>
 
-      <example>
-        <title>Fetching a repository's specific commit on the default branch</title>
-        <para>
-          If the revision you're looking for is in the default branch
-          of the git repository you may omit the
-          <varname>ref</varname> attribute.
-        </para>
-        <programlisting>builtins.fetchGit {
+          <note>
+            <para>
+              It is nice to always specify the branch which a revision
+              belongs to. Without the branch being specified, the
+              fetcher might fail if the default branch changes.
+              Additionally, it can be confusing to try a commit from a
+              non-default branch and see the fetch fail. If the branch
+              is specified the fault is much more obvious.
+            </para>
+          </note>
+        </listitem>
+
+        <listitem>
+          <para>
+            If the revision you're looking for is in the default branch
+            of the git repository you may omit the
+            <varname>ref</varname> attribute.
+          </para>
+          <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
   rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
 }</programlisting>
-      </example>
+        </listitem>
 
-      <example>
-        <title>Fetching a tag</title>
-        <programlisting>builtins.fetchGit {
+        <listitem>
+          <para>To fetch a specific tag:</para>
+          <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
   ref = "refs/tags/1.9";
 }</programlisting>
-      </example>
+        </listitem>
 
-      <example>
-        <title>Fetching the latest version of a remote branch</title>
-        <para>
-          <function>builtins.fetchGit</function> can behave impurely
-           fetch the latest version of a remote branch.
-        </para>
-        <note><para>Nix will refetch the branch in accordance to
-        <xref linkend="conf-tarball-ttl" />.</para></note>
-        <note><para>This behavior is disabled in
-        <emphasis>Pure evaluation mode</emphasis>.</para></note>
+        <listitem>
+          <para>To fetch the latest version of a remote branch:</para>
         <programlisting>builtins.fetchGit {
   url = "ssh://git@github.com/nixos/nix.git";
   ref = "master";
 }</programlisting>
-      </example>
+          <note><para>Nix will refetch the branch in accordance to
+          <xref linkend="conf-tarball-ttl" />.</para></note>
+          <note><para>This behavior is disabled in <emphasis>Pure
+          evaluation mode</emphasis>.</para></note>
+        </listitem>
+      </itemizedlist>
     </listitem>
   </varlistentry>
 
diff --git a/doc/manual/packages/copy-closure.xml b/doc/manual/packages/copy-closure.xml
index 012030e3e..a336fb293 100644
--- a/doc/manual/packages/copy-closure.xml
+++ b/doc/manual/packages/copy-closure.xml
@@ -4,7 +4,7 @@
          version="5.0"
          xml:id="ssec-copy-closure">
 
-<title>Copying Closures Via SSH</title>
+<title>Copying Closures via SSH</title>
 
 <para>The command <command
 linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix
diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index 868b5a66d..aa5df2026 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -5,10 +5,10 @@
          version="5.0"
          xml:id="ssec-s3-substituter">
 
-<title>Serving a Nix store via AWS S3 or S3-compatible Service</title>
+<title>Serving a Nix store via S3</title>
 
 <para>Nix has built-in support for storing and fetching store paths
-from Amazon S3 and S3 compatible services. This uses the same
+from Amazon S3 and S3-compatible services. This uses the same
 <emphasis>binary</emphasis> cache mechanism that Nix usually uses to
 fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
 
@@ -170,13 +170,22 @@ the S3 URL:</para>
 }
 ]]></programlisting>
 
-
-  <example><title>Uploading with a specific credential profile for Amazon S3</title>
-    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;region=eu-west-2' nixpkgs.hello</command></para>
-  </example>
-
-  <example><title>Uploading to an S3-Compatible Binary Cache</title>
-    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
-  </example>
 </section>
+
+<section><title>Examples</title>
+
+<para>To upload with a specific credential profile for Amazon S3:</para>
+
+<screen>
+nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;region=eu-west-2' nixpkgs.hello
+</screen>
+
+<para>To upload to an S3-compatible binary cache:</para>
+
+<screen>
+nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello
+</screen>
+
+</section>
+
 </section>
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index f60af9d24..8faf4b939 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -169,70 +169,76 @@ For instance, `derivation` is also available as `builtins.derivation`.
         A Boolean parameter that specifies whether submodules should be
         checked out. Defaults to `false`.
     
-    <!-- end list -->
+    Here are some examples of how to use `fetchGit`.
     
-        builtins.fetchGit {
-          url = "git@github.com:my-secret/repository.git";
-          ref = "master";
-          rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
-        }
+      - To fetch a private repository over SSH:
+        
+            builtins.fetchGit {
+              url = "git@github.com:my-secret/repository.git";
+              ref = "master";
+              rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+            }
     
-        builtins.fetchGit {
-          url = "https://github.com/NixOS/nix.git";
-          ref = "refs/heads/0.5-release";
-        }
+      - To fetch an arbitrary reference:
+        
+            builtins.fetchGit {
+              url = "https://github.com/NixOS/nix.git";
+              ref = "refs/heads/0.5-release";
+            }
     
-    If the revision you're looking for is in the default branch of the
-    git repository you don't strictly need to specify the branch name in
-    the `ref` attribute.
+      - If the revision you're looking for is in the default branch of
+        the git repository you don't strictly need to specify the branch
+        name in the `ref` attribute.
+        
+        However, if the revision you're looking for is in a future
+        branch for the non-default branch you will need to specify the
+        the `ref` attribute as well.
+        
+            builtins.fetchGit {
+              url = "https://github.com/nixos/nix.git";
+              rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+              ref = "1.11-maintenance";
+            }
+        
+        > **Note**
+        > 
+        > It is nice to always specify the branch which a revision
+        > belongs to. Without the branch being specified, the fetcher
+        > might fail if the default branch changes. Additionally, it can
+        > be confusing to try a commit from a non-default branch and see
+        > the fetch fail. If the branch is specified the fault is much
+        > more obvious.
     
-    However, if the revision you're looking for is in a future branch
-    for the non-default branch you will need to specify the the `ref`
-    attribute as well.
+      - If the revision you're looking for is in the default branch of
+        the git repository you may omit the `ref` attribute.
+        
+            builtins.fetchGit {
+              url = "https://github.com/nixos/nix.git";
+              rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+            }
     
-        builtins.fetchGit {
-          url = "https://github.com/nixos/nix.git";
-          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-          ref = "1.11-maintenance";
-        }
+      - To fetch a specific tag:
+        
+            builtins.fetchGit {
+              url = "https://github.com/nixos/nix.git";
+              ref = "refs/tags/1.9";
+            }
     
-    > **Note**
-    > 
-    > It is nice to always specify the branch which a revision belongs
-    > to. Without the branch being specified, the fetcher might fail if
-    > the default branch changes. Additionally, it can be confusing to
-    > try a commit from a non-default branch and see the fetch fail. If
-    > the branch is specified the fault is much more obvious.
-    
-    If the revision you're looking for is in the default branch of the
-    git repository you may omit the `ref` attribute.
-    
-        builtins.fetchGit {
-          url = "https://github.com/nixos/nix.git";
-          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-        }
-    
-        builtins.fetchGit {
-          url = "https://github.com/nixos/nix.git";
-          ref = "refs/tags/1.9";
-        }
-    
-    `builtins.fetchGit` can behave impurely fetch the latest version of
-    a remote branch.
-    
-    > **Note**
-    > 
-    > Nix will refetch the branch in accordance to
-    > [???](#conf-tarball-ttl).
-    
-    > **Note**
-    > 
-    > This behavior is disabled in *Pure evaluation mode*.
-    
-        builtins.fetchGit {
-          url = "ssh://git@github.com/nixos/nix.git";
-          ref = "master";
-        }
+      - To fetch the latest version of a remote branch:
+        
+            builtins.fetchGit {
+              url = "ssh://git@github.com/nixos/nix.git";
+              ref = "master";
+            }
+        
+        > **Note**
+        > 
+        > Nix will refetch the branch in accordance to
+        > [???](#conf-tarball-ttl).
+        
+        > **Note**
+        > 
+        > This behavior is disabled in *Pure evaluation mode*.
 
   - `builtins.filter` f xs  
     Return a list consisting of the elements of xs for which the
diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md
index 6573862c0..d78b77e36 100644
--- a/doc/manual/src/package-management/copy-closure.md
+++ b/doc/manual/src/package-management/copy-closure.md
@@ -1,4 +1,4 @@
-# Copying Closures Via SSH
+# Copying Closures via SSH
 
 The command `nix-copy-closure` copies a Nix store path along with all
 its dependencies to or from another machine via the SSH protocol. It
diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md
index 4740b8b1c..d96114e3c 100644
--- a/doc/manual/src/package-management/s3-substituter.md
+++ b/doc/manual/src/package-management/s3-substituter.md
@@ -1,7 +1,7 @@
-# Serving a Nix store via AWS S3 or S3-compatible Service
+# Serving a Nix store via S3
 
 Nix has built-in support for storing and fetching store paths from
-Amazon S3 and S3 compatible services. This uses the same *binary* cache
+Amazon S3 and S3-compatible services. This uses the same *binary* cache
 mechanism that Nix usually uses to fetch prebuilt binaries from
 [cache.nixos.org](cache.nixos.org).
 
@@ -124,10 +124,12 @@ Your account will need the following IAM policy to upload to the cache:
       ]
     }
 
-`nix copy --to
-'s3://example-nix-cache?profile=cache-upload&region=eu-west-2'
-nixpkgs.hello`
+## Examples
 
-`nix copy --to
-'s3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
-nixpkgs.hello`
+To upload with a specific credential profile for Amazon S3:
+
+    nix copy --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2' nixpkgs.hello
+
+To upload to an S3-compatible binary cache:
+
+    nix copy --to 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' nixpkgs.hello

From 136fd55bb2f7c4f8e93992c6b662d54ce941a73a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 14:16:46 +0200
Subject: [PATCH 032/882] Get rid of <figure>

---
 doc/manual/packages/profiles.xml              | 29 +++++++++----------
 doc/manual/src/package-management/profiles.md | 15 +++++-----
 2 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/doc/manual/packages/profiles.xml b/doc/manual/packages/profiles.xml
index 15085ab58..2809def24 100644
--- a/doc/manual/packages/profiles.xml
+++ b/doc/manual/packages/profiles.xml
@@ -21,19 +21,16 @@ The long strings prefixed to the directory names are cryptographic
 hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
 a base-32 notation, to be precise.</para></footnote> of
 <emphasis>all</emphasis> inputs involved in building the package —
-sources, dependencies, compiler flags, and so on.  So if two
-packages differ in any way, they end up in different locations in
-the file system, so they don’t interfere with each other.  <xref
-linkend='fig-user-environments' /> shows a part of a typical Nix
-store.</para>
+sources, dependencies, compiler flags, and so on.  So if two packages
+differ in any way, they end up in different locations in the file
+system, so they don’t interfere with each other.  Here is what a part
+of a typical Nix store looks like:</para>
 
-<figure xml:id='fig-user-environments'><title>User environments</title>
-  <mediaobject>
-    <imageobject>
-      <imagedata fileref='../figures/user-environments.png' format='PNG' />
-    </imageobject>
-  </mediaobject>
-</figure>
+<mediaobject>
+  <imageobject>
+    <imagedata fileref='../figures/user-environments.png' format='PNG' />
+  </imageobject>
+</mediaobject>
 
 <para>Of course, you wouldn’t want to type
 
@@ -50,10 +47,10 @@ uses is to create directory trees of symlinks to
 <emphasis>user environments</emphasis> and they are packages
 themselves (though automatically generated by
 <command>nix-env</command>), so they too reside in the Nix store.  For
-instance, in <xref linkend='fig-user-environments' /> the user
-environment <filename>/nix/store/0c1p5z4kda11...-user-env</filename>
-contains a symlink to just Subversion 1.1.2 (arrows in the figure
-indicate symlinks).  This would be what we would obtain if we had done
+instance, in the figure above, the user environment
+<filename>/nix/store/0c1p5z4kda11...-user-env</filename> contains a
+symlink to just Subversion 1.1.2 (arrows in the figure indicate
+symlinks).  This would be what we would obtain if we had done
 
 <screen>
 $ nix-env -i subversion</screen>
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
index 984afca55..9076033d7 100644
--- a/doc/manual/src/package-management/profiles.md
+++ b/doc/manual/src/package-management/profiles.md
@@ -14,10 +14,10 @@ strings prefixed to the directory names are cryptographic hashes\[1\] of
 *all* inputs involved in building the package — sources, dependencies,
 compiler flags, and so on. So if two packages differ in any way, they
 end up in different locations in the file system, so they don’t
-interfere with each other. [figure\_title](#fig-user-environments) shows
-a part of a typical Nix store.
+interfere with each other. Here is what a part of a typical Nix store
+looks like:
 
-![User environments](../figures/user-environments.png)
+![](../figures/user-environments.png)
 
 Of course, you wouldn’t want to type
 
@@ -30,11 +30,10 @@ package we want to use, but this is not very convenient since changing
 Nix uses is to create directory trees of symlinks to *activated*
 packages. These are called *user environments* and they are packages
 themselves (though automatically generated by `nix-env`), so they too
-reside in the Nix store. For instance, in
-[figure\_title](#fig-user-environments) the user environment
-`/nix/store/0c1p5z4kda11...-user-env` contains a symlink to just
-Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be
-what we would obtain if we had done
+reside in the Nix store. For instance, in the figure above, the user
+environment `/nix/store/0c1p5z4kda11...-user-env` contains a symlink to
+just Subversion 1.1.2 (arrows in the figure indicate symlinks). This
+would be what we would obtain if we had done
 
     $ nix-env -i subversion
 

From ee051084723333fc5889c604c829669800e8b43c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 14:20:54 +0200
Subject: [PATCH 033/882] <simplesect> -> <section>

Pandoc silently ignores <simplesect>...
---
 .../expressions/language-constructs.xml       | 32 ++++++------
 doc/manual/expressions/language-values.xml    | 12 ++---
 doc/manual/installation/multi-user.xml        | 12 ++---
 doc/manual/introduction/about-nix.xml         | 52 +++++++++----------
 .../src/expressions/language-constructs.md    | 16 ++++++
 doc/manual/src/expressions/language-values.md |  6 +++
 doc/manual/src/installation/multi-user.md     |  6 +++
 7 files changed, 82 insertions(+), 54 deletions(-)

diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 4d316609c..82d3afed1 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -6,7 +6,7 @@
 
 <title>Language Constructs</title>
 
-<simplesect><title>Recursive sets</title>
+<section><title>Recursive sets</title>
 
 <para>Recursive sets are just normal sets, but the attributes can
 refer to each other.  For example,
@@ -38,10 +38,10 @@ does not terminate<footnote><para>Actually, Nix detects infinite
 recursion in this case and aborts (<quote>infinite recursion
 encountered</quote>).</para></footnote>.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect xml:id="sect-let-expressions"><title>Let-expressions</title>
+<section xml:id="sect-let-expressions"><title>Let-expressions</title>
 
 <para>A let-expression allows you to define local variables for an
 expression.  For instance,
@@ -56,10 +56,10 @@ evaluates to <literal>"foobar"</literal>.
 
 </para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Inheriting attributes</title>
+<section><title>Inheriting attributes</title>
 
 <para>When defining a set or in a let-expression it is often convenient to copy variables
 from the surrounding lexical scope (e.g., when you want to propagate
@@ -129,10 +129,10 @@ a = src-set.a; b = src-set.b; c = src-set.c;
 when used while defining local variables in a let-expression or
 while defining a set.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect xml:id="ss-functions"><title>Functions</title>
+<section xml:id="ss-functions"><title>Functions</title>
 
 <para>Functions have the following form:
 
@@ -248,10 +248,10 @@ in concat { x = "foo"; y = "bar"; }</programlisting>
 
 </para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Conditionals</title>
+<section><title>Conditionals</title>
 
 <para>Conditionals look like this:
 
@@ -262,10 +262,10 @@ where <replaceable>e1</replaceable> is an expression that should
 evaluate to a Boolean value (<literal>true</literal> or
 <literal>false</literal>).</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Assertions</title>
+<section><title>Assertions</title>
 
 <para>Assertions are generally used to check that certain requirements
 on or between features and dependencies hold.  They look like this:
@@ -349,11 +349,11 @@ stdenv.mkDerivation {
 
 </orderedlist>
 
-</simplesect>
+</section>
 
 
 
-<simplesect><title>With-expressions</title>
+<section><title>With-expressions</title>
 
 <para>A <emphasis>with-expression</emphasis>,
 
@@ -394,16 +394,16 @@ let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...</programlisting>
 
 </para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Comments</title>
+<section><title>Comments</title>
 
 <para>Comments can be single-line, started with a <literal>#</literal>
 character, or inline/multi-line, enclosed within <literal>/*
 ... */</literal>.</para>
 
-</simplesect>
+</section>
 
 
 </section>
diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml
index 4a72c67a8..6c0fcbacb 100644
--- a/doc/manual/expressions/language-values.xml
+++ b/doc/manual/expressions/language-values.xml
@@ -7,7 +7,7 @@
 <title>Values</title>
 
 
-<simplesect><title>Simple Values</title>
+<section><title>Simple Values</title>
 
 <para>Nix has the following basic data types:
 
@@ -193,10 +193,10 @@ stdenv.mkDerivation {
 
 </para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Lists</title>
+<section><title>Lists</title>
 
 <para>Lists are formed by enclosing a whitespace-separated list of
 values between square brackets.  For example,
@@ -217,10 +217,10 @@ function and the fifth being a set.</para>
 <para>Note that lists are only lazy in values, and they are strict in length.
 </para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Sets</title>
+<section><title>Sets</title>
 
 <para>Sets are really the core of the language, since ultimately the
 Nix language is all about creating derivations, which are really just
@@ -307,7 +307,7 @@ a form of object-oriented programming, for example.
 
 </para>
 
-</simplesect>
+</section>
 
 
 </section>
diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml
index 331b5f128..835bd3a52 100644
--- a/doc/manual/installation/multi-user.xml
+++ b/doc/manual/installation/multi-user.xml
@@ -30,7 +30,7 @@ arbitrary Nix expressions, they may not get pre-built
 binaries.</para></note>
 
 
-<simplesect>
+<section>
 
 <title>Setting up the build users</title>
 
@@ -52,10 +52,10 @@ This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if
 you expect to do many builds at the same time.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect>
+<section>
 
 <title>Running the daemon</title>
 
@@ -78,10 +78,10 @@ export NIX_REMOTE=daemon</programlisting>
 
 into the users’ login scripts.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect>
+<section>
 
 <title>Restricting access</title>
 
@@ -101,7 +101,7 @@ cannot connect to the Unix domain socket
 <filename>/nix/var/nix/daemon-socket/socket</filename>, so they cannot
 perform Nix operations.</para>
 
-</simplesect>
+</section>
 
 
 </section>
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index ec6f1ca3f..dd09e2283 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -25,7 +25,7 @@ of the package’s build dependency graph).  This enables many powerful
 features.</para>
 
 
-<simplesect><title>Multiple versions</title>
+<section><title>Multiple versions</title>
 
 <para>You can have multiple versions or variants of a package
 installed at the same time.  This is especially important when
@@ -39,10 +39,10 @@ uninstalling an application cannot break other applications, since
 these operations never “destructively” update or delete files that are
 used by other packages.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Complete dependencies</title>
+<section><title>Complete dependencies</title>
 
 <para>Nix helps you make sure that package dependency specifications
 are complete.  In general, when you’re making a package for a package
@@ -68,10 +68,10 @@ scanning binaries for the hash parts of Nix store paths (such as
 <literal>r8vvq9kq…</literal>).  This sounds risky, but it works
 extremely well.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Multi-user support</title>
+<section><title>Multi-user support</title>
 
 <para>Nix has multi-user support.  This means that non-privileged
 users can securely install software.  Each user can have a different
@@ -82,10 +82,10 @@ package won’t be built or downloaded a second time.  At the same time,
 it is not possible for one user to inject a Trojan horse into a
 package that might be used by another user.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Atomic upgrades and rollbacks</title>
+<section><title>Atomic upgrades and rollbacks</title>
 
 <para>Since package management operations never overwrite packages in
 the Nix store but just add new versions in different paths, they are
@@ -103,10 +103,10 @@ $ nix-env --upgrade <replaceable>some-packages</replaceable>
 $ nix-env --rollback
 </screen>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Garbage collection</title>
+<section><title>Garbage collection</title>
 
 <para>When you uninstall a package like this…
 
@@ -126,10 +126,10 @@ $ nix-collect-garbage
 This deletes all packages that aren’t in use by any user profile or by
 a currently running program.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Functional package language</title>
+<section><title>Functional package language</title>
 
 <para>Packages are built from <emphasis>Nix expressions</emphasis>,
 which is a simple functional language.  A Nix expression describes
@@ -145,10 +145,10 @@ function and call it any number of times with the appropriate
 arguments.  Due to the hashing scheme, variants don’t conflict with
 each other in the Nix store.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Transparent source/binary deployment</title>
+<section><title>Transparent source/binary deployment</title>
 
 <para>Nix expressions generally describe how to build a package from
 source, so an installation action like
@@ -172,31 +172,31 @@ Nix would first check if the file
 if so, fetch the pre-built binary referenced from there; otherwise, it
 would fall back to building from source.</para>
 
-</simplesect>
+</section>
 
 
 <!--
-<simplesect><title>Binary patching</title>
+<section><title>Binary patching</title>
 
 <para>In addition to downloading binaries automatically if they’re
 available, Nix can download binary deltas that patch an existing
 package in the Nix store into a new version.  This speeds up
 upgrades.</para>
 
-</simplesect>
+</section>
 -->
 
 
-<simplesect><title>Nix Packages collection</title>
+<section><title>Nix Packages collection</title>
 
 <para>We provide a large set of Nix expressions containing hundreds of
 existing Unix packages, the <emphasis>Nix Packages
 collection</emphasis> (Nixpkgs).</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Managing build environments</title>
+<section><title>Managing build environments</title>
 
 <para>Nix is extremely useful for developers as it makes it easy to
 automatically set up the build environment for a package. Given a
@@ -232,17 +232,17 @@ specifications, Nix makes an excellent basis for <a
 href="[%root%]hydra">a continuous build system</a>.</para>
 -->
 
-</simplesect>
+</section>
 
 
-<simplesect><title>Portability</title>
+<section><title>Portability</title>
 
 <para>Nix runs on Linux and macOS.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>NixOS</title>
+<section><title>NixOS</title>
 
 <para>NixOS is a Linux distribution based on Nix.  It uses Nix not
 just for package management but also to manage the system
@@ -253,16 +253,16 @@ earlier state.  Also, users can install software without root
 privileges.  For more information and downloads, see the <link
 xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 
-</simplesect>
+</section>
 
 
-<simplesect><title>License</title>
+<section><title>License</title>
 
 <para>Nix is released under the terms of the <link
 xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
 LGPLv2.1 or (at your option) any later version</link>.</para>
 
-</simplesect>
+</section>
 
 
 </chapter>
diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/expressions/language-constructs.md
index 121e4d998..20d003348 100644
--- a/doc/manual/src/expressions/language-constructs.md
+++ b/doc/manual/src/expressions/language-constructs.md
@@ -1,5 +1,7 @@
 # Language Constructs
 
+## Recursive sets
+
 Recursive sets are just normal sets, but the attributes can refer to
 each other. For example,
 
@@ -24,6 +26,8 @@ example,
 
 does not terminate\[1\].
 
+## Let-expressions
+
 A let-expression allows you to define local variables for an expression.
 For instance,
 
@@ -34,6 +38,8 @@ For instance,
 
 evaluates to `"foobar"`.
 
+## Inheriting attributes
+
 When defining a set or in a let-expression it is often convenient to
 copy variables from the surrounding lexical scope (e.g., when you want
 to propagate attributes). This can be shortened using the `inherit`
@@ -95,6 +101,8 @@ is equivalent to
 when used while defining local variables in a let-expression or while
 defining a set.
 
+## Functions
+
 Functions have the following form:
 
     pattern: body
@@ -187,6 +195,8 @@ you can bind them to an attribute, e.g.,
     let concat = { x, y }: x + y;
     in concat { x = "foo"; y = "bar"; }
 
+## Conditionals
+
 Conditionals look like this:
 
     if e1 then e2 else e3
@@ -194,6 +204,8 @@ Conditionals look like this:
 where e1 is an expression that should evaluate to a Boolean value
 (`true` or `false`).
 
+## Assertions
+
 Assertions are generally used to check that certain requirements on or
 between features and dependencies hold. They look like this:
 
@@ -257,6 +269,8 @@ The points of interest are:
     non-`null` value was passed. This prevents an unnecessary rebuild of
     Subversion if OpenSSL changes.
 
+## With-expressions
+
 A *with-expression*,
 
     with e1; e2
@@ -285,6 +299,8 @@ establishes the same scope as
 
     let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...
 
+## Comments
+
 Comments can be single-line, started with a `#` character, or
 inline/multi-line, enclosed within `/*
 ... */`.
diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md
index fac0be47e..8ffe2f0f9 100644
--- a/doc/manual/src/expressions/language-values.md
+++ b/doc/manual/src/expressions/language-values.md
@@ -1,5 +1,7 @@
 # Values
 
+## Simple Values
+
 Nix has the following basic data types:
 
   - *Strings* can be written in three ways.
@@ -127,6 +129,8 @@ Nix has the following basic data types:
 
   - The null value, denoted as `null`.
 
+## Lists
+
 Lists are formed by enclosing a whitespace-separated list of values
 between square brackets. For example,
 
@@ -143,6 +147,8 @@ function and the fifth being a set.
 
 Note that lists are only lazy in values, and they are strict in length.
 
+## Sets
+
 Sets are really the core of the language, since ultimately the Nix
 language is all about creating derivations, which are really just sets
 of attributes to be passed to build scripts.
diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/src/installation/multi-user.md
index 6493e717b..4a861691d 100644
--- a/doc/manual/src/installation/multi-user.md
+++ b/doc/manual/src/installation/multi-user.md
@@ -20,6 +20,8 @@ Nix store/database that performs the operation.
 > caches. So while unprivileged users may install packages from
 > arbitrary Nix expressions, they may not get pre-built binaries.
 
+## Setting up the build users
+
 The *build users* are the special UIDs under which builds are performed.
 They should all be members of the *build users group* `nixbld`. This
 group should have no other members. The build users should not be
@@ -35,6 +37,8 @@ This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if you
 expect to do many builds at the same time.
 
+## Running the daemon
+
 The [Nix daemon](#sec-nix-daemon) should be started as follows (as
 `root`):
 
@@ -50,6 +54,8 @@ should put a line like
 
 into the users’ login scripts.
 
+## Restricting access
+
 To limit which users can perform Nix operations, you can use the
 permissions on the directory `/nix/var/nix/daemon-socket`. For instance,
 if you want to restrict the use of Nix to the members of a group called

From 802150f987e720452920a3d1993c3b4b36861116 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 14:28:05 +0200
Subject: [PATCH 034/882] <replaceable> -> <emphasis>

Pandoc doesn't know <replaceable> so let's force it to be rendered as
italics.
---
 .../advanced-topics/distributed-builds.xml    |   4 +-
 doc/manual/command-ref/conf-file.xml          |  34 +-
 doc/manual/command-ref/env-common.xml         |  18 +-
 doc/manual/command-ref/nix-build.xml          |  24 +-
 doc/manual/command-ref/nix-channel.xml        |  32 +-
 .../command-ref/nix-collect-garbage.xml       |   6 +-
 doc/manual/command-ref/nix-copy-closure.xml   |  16 +-
 doc/manual/command-ref/nix-env.xml            | 120 ++---
 doc/manual/command-ref/nix-hash.xml           |  24 +-
 doc/manual/command-ref/nix-instantiate.xml    |  28 +-
 doc/manual/command-ref/nix-prefetch-url.xml   |  30 +-
 doc/manual/command-ref/nix-shell.xml          |  48 +-
 doc/manual/command-ref/nix-store.xml          | 122 ++---
 doc/manual/command-ref/opt-common-syn.xml     |  16 +-
 doc/manual/command-ref/opt-common.xml         |  46 +-
 doc/manual/command-ref/opt-inst-syn.xml       |   2 +-
 .../expressions/advanced-attributes.xml       |  18 +-
 doc/manual/expressions/build-script.xml       |   2 +-
 doc/manual/expressions/builtins.xml           | 420 +++++++++---------
 doc/manual/expressions/expression-syntax.xml  |  10 +-
 doc/manual/expressions/generic-builder.xml    |   2 +-
 .../expressions/language-constructs.xml       |  24 +-
 doc/manual/expressions/language-operators.xml |  78 ++--
 doc/manual/expressions/language-values.xml    |   8 +-
 .../expressions/simple-building-testing.xml   |   2 +-
 doc/manual/installation/building-source.xml   |  10 +-
 doc/manual/installation/env-variables.xml     |   6 +-
 doc/manual/installation/installing-binary.xml |   2 +-
 doc/manual/installation/single-user.xml       |   4 +-
 doc/manual/introduction/about-nix.xml         |   2 +-
 doc/manual/introduction/quick-start.xml       |   2 +-
 doc/manual/packages/basic-package-mgmt.xml    |   4 +-
 doc/manual/packages/channels.xml              |   2 +-
 .../packages/garbage-collector-roots.xml      |   4 +-
 doc/manual/packages/profiles.xml              |   2 +-
 doc/manual/release-notes/rl-0.10.xml          |  16 +-
 doc/manual/release-notes/rl-0.12.xml          |  16 +-
 doc/manual/release-notes/rl-0.13.xml          |   4 +-
 doc/manual/release-notes/rl-0.16.xml          |   8 +-
 doc/manual/release-notes/rl-0.6.xml           |   8 +-
 doc/manual/release-notes/rl-0.8.xml           |   4 +-
 doc/manual/release-notes/rl-1.1.xml           |   4 +-
 doc/manual/release-notes/rl-1.11.xml          |   8 +-
 doc/manual/release-notes/rl-1.2.xml           |   2 +-
 doc/manual/release-notes/rl-1.4.xml           |   4 +-
 doc/manual/release-notes/rl-1.6.1.xml         |   8 +-
 doc/manual/release-notes/rl-1.7.xml           |  12 +-
 doc/manual/release-notes/rl-1.8.xml           |   4 +-
 doc/manual/release-notes/rl-2.0.xml           |  12 +-
 doc/manual/src/command-ref/conf-file.md       |  10 +-
 doc/manual/src/command-ref/nix-build.md       |  12 +-
 doc/manual/src/command-ref/nix-channel.md     |  21 +-
 .../src/command-ref/nix-collect-garbage.md    |   2 +-
 doc/manual/src/command-ref/nix-env.md         |  61 +--
 doc/manual/src/command-ref/nix-hash.md        |  14 +-
 doc/manual/src/command-ref/nix-instantiate.md |   8 +-
 .../src/command-ref/nix-prefetch-url.md       |  22 +-
 doc/manual/src/command-ref/nix-shell.md       |  30 +-
 doc/manual/src/command-ref/nix-store.md       |  80 ++--
 doc/manual/src/command-ref/opt-common.md      |  26 +-
 .../src/expressions/advanced-attributes.md    |  10 +-
 doc/manual/src/expressions/builtins.md        | 372 ++++++++--------
 .../src/expressions/expression-syntax.md      |   2 +-
 .../src/expressions/language-constructs.md    |  14 +-
 .../src/expressions/language-operators.md     |  48 +-
 .../src/installation/building-source.md       |   2 +-
 .../package-management/basic-package-mgmt.md  |   3 +-
 doc/manual/src/release-notes/rl-0.10.md       |   2 +-
 doc/manual/src/release-notes/rl-0.12.md       |   4 +-
 doc/manual/src/release-notes/rl-0.13.md       |   4 +-
 doc/manual/src/release-notes/rl-0.16.md       |   2 +-
 doc/manual/src/release-notes/rl-0.6.md        |   4 +-
 doc/manual/src/release-notes/rl-1.11.md       |   2 +-
 doc/manual/src/release-notes/rl-1.4.md        |   2 +-
 doc/manual/src/release-notes/rl-1.6.1.md      |   4 +-
 doc/manual/src/release-notes/rl-1.7.md        |   6 +-
 76 files changed, 1029 insertions(+), 1020 deletions(-)

diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index a649b6f8d..ec9e98e77 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -86,7 +86,7 @@ To leave a field at its default, set it to <literal>-</literal>.
 <orderedlist>
 
   <listitem><para>The URI of the remote store in the format
-  <literal>ssh://[<replaceable>username</replaceable>@]<replaceable>hostname</replaceable></literal>,
+  <literal>ssh://[<emphasis>username</emphasis>@]<emphasis>hostname</emphasis></literal>,
   e.g. <literal>ssh://nix@mac</literal> or
   <literal>ssh://mac</literal>. For backward compatibility,
   <literal>ssh://</literal> may be omitted. The hostname may be an
@@ -171,7 +171,7 @@ builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd
 
 Finally, remote builders can be configured in a separate configuration
 file included in <option>builders</option> via the syntax
-<literal>@<replaceable>file</replaceable></literal>. For example,
+<literal>@<emphasis>file</emphasis></literal>. For example,
 
 <programlisting>
 builders = @/etc/nix/machines
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 4c103f505..62cc117b6 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -26,7 +26,7 @@
   <listitem>
     <para>
       The system-wide configuration file
-      <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+      <filename><emphasis>sysconfdir</emphasis>/nix/nix.conf</filename>
       (i.e. <filename>/etc/nix/nix.conf</filename> on most systems),
       or <filename>$NIX_CONF_DIR/nix.conf</filename> if
       <literal>NIX_CONF_DIR</literal> is set. Values loaded in this
@@ -52,11 +52,11 @@
 </itemizedlist>
 
 <para>The configuration files consist of
-<literal><replaceable>name</replaceable> =
-<replaceable>value</replaceable></literal> pairs, one per line. Other
+<literal><emphasis>name</emphasis> =
+<emphasis>value</emphasis></literal> pairs, one per line. Other
 files can be included with a line like <literal>include
-<replaceable>path</replaceable></literal>, where
-<replaceable>path</replaceable> is interpreted relative to the current
+<emphasis>path</emphasis></literal>, where
+<emphasis>path</emphasis> is interpreted relative to the current
 conf file and a missing file is an error unless
 <literal>!include</literal> is used instead.
 Comments start with a <literal>#</literal> character.  Here is an
@@ -244,7 +244,7 @@ false</literal>.</para>
     instance, in Nixpkgs, if the derivation attribute
     <varname>enableParallelBuilding</varname> is set to
     <literal>true</literal>, the builder passes the
-    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+    <option>-j<emphasis>N</emphasis></option> flag to GNU Make.
     It can be overridden using the <option
     linkend='opt-cores'>--cores</option> command line switch and
     defaults to <literal>1</literal>.  The value <literal>0</literal>
@@ -383,10 +383,10 @@ false</literal>.</para>
     <function>builtins.fetchurl</function> to obtain files by
     hash. The default is
     <literal>http://tarballs.nixos.org/</literal>. Given a hash type
-    <replaceable>ht</replaceable> and a base-16 hash
-    <replaceable>h</replaceable>, Nix will try to download the file
+    <emphasis>ht</emphasis> and a base-16 hash
+    <emphasis>h</emphasis>, Nix will try to download the file
     from
-    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
+    <literal>hashed-mirror/<emphasis>ht</emphasis>/<emphasis>h</emphasis></literal>.
     This allows files to be downloaded even if they have disappeared
     from their original URI. For example, given the default mirror
     <literal>http://tarballs.nixos.org/</literal>, when building the derivation
@@ -421,7 +421,7 @@ builtins.fetchurl {
     output and error of its builder) to the directory
     <filename>/nix/var/log/nix/drvs</filename>.  The build log can be
     retrieved using the command <command>nix-store -l
-    <replaceable>path</replaceable></command>.</para></listitem>
+    <emphasis>path</emphasis></command>.</para></listitem>
 
   </varlistentry>
 
@@ -592,9 +592,9 @@ builtins.fetchurl {
     accounts in the following format:
 
 <screen>
-machine <replaceable>my-machine</replaceable>
-login <replaceable>my-username</replaceable>
-password <replaceable>my-password</replaceable>
+machine <emphasis>my-machine</emphasis>
+login <emphasis>my-username</emphasis>
+password <emphasis>my-password</emphasis>
 </screen>
 
     For the exact syntax, see <link
@@ -837,13 +837,13 @@ password <replaceable>my-password</replaceable>
 
     <listitem><para>A list of paths bind-mounted into Nix sandbox
     environments. You can use the syntax
-    <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal>
+    <literal><emphasis>target</emphasis>=<emphasis>source</emphasis></literal>
     to mount a path in a different location in the sandbox; for
     instance, <literal>/bin=/nix-bin</literal> will mount the path
     <literal>/nix-bin</literal> as <literal>/bin</literal> inside the
-    sandbox. If <replaceable>source</replaceable> is followed by
+    sandbox. If <emphasis>source</emphasis> is followed by
     <literal>?</literal>, then it is not an error if
-    <replaceable>source</replaceable> does not exist; for example,
+    <emphasis>source</emphasis> does not exist; for example,
     <literal>/dev/nvidiactl?</literal> specifies that
     <filename>/dev/nvidiactl</filename> will only be mounted in the
     sandbox if it exists in the host filesystem.</para>
@@ -1035,7 +1035,7 @@ function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
     <listitem><para>A list of URLs of substituters, separated by
     whitespace.  These are not used by default, but can be enabled by
     users of the Nix daemon by specifying <literal>--option
-    substituters <replaceable>urls</replaceable></literal> on the
+    substituters <emphasis>urls</emphasis></literal> on the
     command line.  Unprivileged users are only allowed to pass a
     subset of the URLs listed in <literal>substituters</literal> and
     <literal>trusted-substituters</literal>.</para></listitem>
diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index 3196cbbd2..ac40fccf7 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -25,7 +25,7 @@
 
     <para>A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e.,
-    <literal>&lt;<replaceable>path</replaceable>></literal>).  For
+    <literal>&lt;<emphasis>path</emphasis>></literal>).  For
     instance, the value
 
     <screen>
@@ -40,10 +40,10 @@
 nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
 
     will cause Nix to search for
-    <literal>&lt;nixpkgs/<replaceable>path</replaceable>></literal> in
-    <filename>/home/eelco/Dev/nixpkgs-branch/<replaceable>path</replaceable></filename>
+    <literal>&lt;nixpkgs/<emphasis>path</emphasis>></literal> in
+    <filename>/home/eelco/Dev/nixpkgs-branch/<emphasis>path</emphasis></filename>
     and
-    <filename>/etc/nixos/nixpkgs/<replaceable>path</replaceable></filename>.</para>
+    <filename>/etc/nixos/nixpkgs/<emphasis>path</emphasis></filename>.</para>
 
     <para>If a path in the Nix search path starts with
     <literal>http://</literal> or <literal>https://</literal>, it is
@@ -105,7 +105,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry><term><literal>NIX_STORE_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix store (default
-  <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
+  <filename><emphasis>prefix</emphasis>/store</filename>).</para></listitem>
 
 </varlistentry>
 
@@ -114,7 +114,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
   <listitem><para>Overrides the location of the Nix static data
   directory (default
-  <filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem>
+  <filename><emphasis>prefix</emphasis>/share</filename>).</para></listitem>
 
 </varlistentry>
 
@@ -122,7 +122,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry><term><literal>NIX_LOG_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix log directory
-  (default <filename><replaceable>prefix</replaceable>/var/log/nix</filename>).</para></listitem>
+  (default <filename><emphasis>prefix</emphasis>/var/log/nix</filename>).</para></listitem>
 
 </varlistentry>
 
@@ -130,7 +130,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry><term><literal>NIX_STATE_DIR</literal></term>
 
   <listitem><para>Overrides the location of the Nix state directory
-  (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
+  (default <filename><emphasis>prefix</emphasis>/var/nix</filename>).</para></listitem>
 
 </varlistentry>
 
@@ -139,7 +139,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
   <listitem><para>Overrides the location of the system Nix configuration
   directory (default
-  <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
+  <filename><emphasis>prefix</emphasis>/etc/nix</filename>).</para></listitem>
 
 </varlistentry>
 
diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml
index 886d25910..ec9145143 100644
--- a/doc/manual/command-ref/nix-build.xml
+++ b/doc/manual/command-ref/nix-build.xml
@@ -20,14 +20,14 @@
   <cmdsynopsis>
     <command>nix-build</command>
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-    <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
-    <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
+    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
+    <arg><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--attr</option></arg>
         <arg choice='plain'><option>-A</option></arg>
       </group>
-      <replaceable>attrPath</replaceable>
+      <emphasis>attrPath</emphasis>
     </arg>
     <arg><option>--no-out-link</option></arg>
     <arg><option>--dry-run</option></arg>
@@ -36,16 +36,16 @@
         <arg choice='plain'><option>--out-link</option></arg>
         <arg choice='plain'><option>-o</option></arg>
       </group>
-      <replaceable>outlink</replaceable>
+      <emphasis>outlink</emphasis>
     </arg>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
 <refsection><title>Description</title>
 
 <para>The <command>nix-build</command> command builds the derivations
-described by the Nix expressions in <replaceable>paths</replaceable>.
+described by the Nix expressions in <emphasis>paths</emphasis>.
 If the build succeeds, it places a symlink to the result in the
 current directory.  The symlink is called <filename>result</filename>.
 If there are multiple Nix expressions, or the Nix expressions evaluate
@@ -53,11 +53,11 @@ to multiple derivations, multiple sequentially numbered symlinks are
 created (<filename>result</filename>, <filename>result-2</filename>,
 and so on).</para>
 
-<para>If no <replaceable>paths</replaceable> are specified, then
+<para>If no <emphasis>paths</emphasis> are specified, then
 <command>nix-build</command> will use <filename>default.nix</filename>
 in the current directory, if it exists.</para>
 
-<para>If an element of <replaceable>paths</replaceable> starts with
+<para>If an element of <emphasis>paths</emphasis> starts with
 <literal>http://</literal> or <literal>https://</literal>, it is
 interpreted as the URL of a tarball that will be downloaded and
 unpacked to a temporary location. The tarball must include a single
@@ -104,11 +104,11 @@ also <xref linkend="sec-common-options" />.</phrase></para>
   </varlistentry>
 
   <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /
-  <option>-o</option> <replaceable>outlink</replaceable></term>
+  <option>-o</option> <emphasis>outlink</emphasis></term>
 
     <listitem><para>Change the name of the symlink to the output path
     created from <filename>result</filename> to
-    <replaceable>outlink</replaceable>.</para></listitem>
+    <emphasis>outlink</emphasis>.</para></listitem>
 
   </varlistentry>
 
@@ -131,7 +131,7 @@ store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
 /nix/store/d18hyl92g30l...-firefox-1.5.0.7
 
 $ ls -l result
-lrwxrwxrwx  <replaceable>...</replaceable>  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
+lrwxrwxrwx  <emphasis>...</emphasis>  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
 
 $ ls ./result/bin/
 firefox  firefox-config</screen>
@@ -143,7 +143,7 @@ You can also build all outputs:
 $ nix-build '&lt;nixpkgs>' -A openssl.all
 </screen>
 This will create a symlink for each output named
-<filename>result-<replaceable>outputname</replaceable></filename>.
+<filename>result-<emphasis>outputname</emphasis></filename>.
 The suffix is omitted if the output name is <literal>out</literal>.
 So if <literal>openssl</literal> has outputs <literal>out</literal>,
 <literal>bin</literal> and <literal>man</literal>,
diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml
index ebcf56aff..2abeca0a0 100644
--- a/doc/manual/command-ref/nix-channel.xml
+++ b/doc/manual/command-ref/nix-channel.xml
@@ -20,11 +20,11 @@
   <cmdsynopsis>
     <command>nix-channel</command>
     <group choice='req'>
-      <arg choice='plain'><option>--add</option> <replaceable>url</replaceable> <arg choice='opt'><replaceable>name</replaceable></arg></arg>
-      <arg choice='plain'><option>--remove</option> <replaceable>name</replaceable></arg>
+      <arg choice='plain'><option>--add</option> <emphasis>url</emphasis> <arg choice='opt'><emphasis>name</emphasis></arg></arg>
+      <arg choice='plain'><option>--remove</option> <emphasis>name</emphasis></arg>
       <arg choice='plain'><option>--list</option></arg>
-      <arg choice='plain'><option>--update</option> <arg rep='repeat'><replaceable>names</replaceable></arg></arg>
-      <arg choice='plain'><option>--rollback</option> <arg choice='opt'><replaceable>generation</replaceable></arg></arg>
+      <arg choice='plain'><option>--update</option> <arg rep='repeat'><emphasis>names</emphasis></arg></arg>
+      <arg choice='plain'><option>--rollback</option> <arg choice='opt'><emphasis>generation</emphasis></arg></arg>
     </group>
   </cmdsynopsis>
 </refsynopsisdiv>
@@ -44,22 +44,22 @@ xlink:href="https://nixos.org/channels" />.</para>
 
 <variablelist>
 
-  <varlistentry><term><option>--add</option> <replaceable>url</replaceable> [<replaceable>name</replaceable>]</term>
+  <varlistentry><term><option>--add</option> <emphasis>url</emphasis> [<emphasis>name</emphasis>]</term>
 
     <listitem><para>Adds a channel named
-    <replaceable>name</replaceable> with URL
-    <replaceable>url</replaceable> to the list of subscribed channels.
-    If <replaceable>name</replaceable> is omitted, it defaults to the
-    last component of <replaceable>url</replaceable>, with the
+    <emphasis>name</emphasis> with URL
+    <emphasis>url</emphasis> to the list of subscribed channels.
+    If <emphasis>name</emphasis> is omitted, it defaults to the
+    last component of <emphasis>url</emphasis>, with the
     suffixes <literal>-stable</literal> or
     <literal>-unstable</literal> removed.</para></listitem>
 
   </varlistentry>
 
-  <varlistentry><term><option>--remove</option> <replaceable>name</replaceable></term>
+  <varlistentry><term><option>--remove</option> <emphasis>name</emphasis></term>
 
     <listitem><para>Removes the channel named
-    <replaceable>name</replaceable> from the list of subscribed
+    <emphasis>name</emphasis> from the list of subscribed
     channels.</para></listitem>
 
   </varlistentry>
@@ -71,18 +71,18 @@ xlink:href="https://nixos.org/channels" />.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--update</option> [<replaceable>names</replaceable>…]</term>
+  <varlistentry><term><option>--update</option> [<emphasis>names</emphasis>…]</term>
 
     <listitem><para>Downloads the Nix expressions of all subscribed
     channels (or only those included in
-    <replaceable>names</replaceable> if specified) and makes them the
+    <emphasis>names</emphasis> if specified) and makes them the
     default for <command>nix-env</command> operations (by symlinking
     them from the directory
     <filename>~/.nix-defexpr</filename>).</para></listitem>
 
   </varlistentry>
 
-  <varlistentry><term><option>--rollback</option> [<replaceable>generation</replaceable>]</term>
+  <varlistentry><term><option>--rollback</option> [<emphasis>generation</emphasis>]</term>
 
     <listitem><para>Reverts the previous call to <command>nix-channel
     --update</command>. Optionally, you can specify a specific channel
@@ -130,7 +130,7 @@ $ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 
 <variablelist>
 
-  <varlistentry><term><filename>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels</filename></term>
+  <varlistentry><term><filename>/nix/var/nix/profiles/per-user/<emphasis>username</emphasis>/channels</filename></term>
 
     <listitem><para><command>nix-channel</command> uses a
     <command>nix-env</command> profile to keep track of previous
@@ -145,7 +145,7 @@ $ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
   <varlistentry><term><filename>~/.nix-defexpr/channels</filename></term>
 
     <listitem><para>This is a symlink to
-    <filename>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels</filename>. It
+    <filename>/nix/var/nix/profiles/per-user/<emphasis>username</emphasis>/channels</filename>. It
     ensures that <command>nix-env</command> can find your channels. In
     a multi-user installation, you may also have
     <filename>~/.nix-defexpr/channels_root</filename>, which links to
diff --git a/doc/manual/command-ref/nix-collect-garbage.xml b/doc/manual/command-ref/nix-collect-garbage.xml
index 43e068796..cbcb5add5 100644
--- a/doc/manual/command-ref/nix-collect-garbage.xml
+++ b/doc/manual/command-ref/nix-collect-garbage.xml
@@ -21,8 +21,8 @@
     <command>nix-collect-garbage</command>
     <arg><option>--delete-old</option></arg>
     <arg><option>-d</option></arg>
-    <arg><option>--delete-older-than</option> <replaceable>period</replaceable></arg>
-    <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
+    <arg><option>--delete-older-than</option> <emphasis>period</emphasis></arg>
+    <arg><option>--max-freed</option> <emphasis>bytes</emphasis></arg>
     <arg><option>--dry-run</option></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
@@ -39,7 +39,7 @@ which deletes all old generations of all profiles in
 <literal>nix-env --delete-generations old</literal> on all profiles
 (of course, this makes rollbacks to previous configurations
 impossible); and
-<option>--delete-older-than</option> <replaceable>period</replaceable>,
+<option>--delete-older-than</option> <emphasis>period</emphasis>,
 where period is a value such as <literal>30d</literal>, which deletes
 all generations older than the specified number of days in all profiles
 in <filename>/nix/var/nix/profiles</filename> (except for the generations
diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml
index 8c07984fb..6fc8c312b 100644
--- a/doc/manual/command-ref/nix-copy-closure.xml
+++ b/doc/manual/command-ref/nix-copy-closure.xml
@@ -33,9 +33,9 @@
     </group>
     <arg><option>-v</option></arg>
     <arg choice='plain'>
-      <replaceable>user@</replaceable><replaceable>machine</replaceable>
+      <emphasis>user@</emphasis><emphasis>machine</emphasis>
     </arg>
-    <arg choice='plain'><replaceable>paths</replaceable></arg>
+    <arg choice='plain'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
@@ -44,13 +44,13 @@
 
 <para><command>nix-copy-closure</command> gives you an easy and
 efficient way to exchange software between machines.  Given one or
-more Nix store <replaceable>paths</replaceable> on the local
+more Nix store <emphasis>paths</emphasis> on the local
 machine, <command>nix-copy-closure</command> computes the closure of
 those paths (i.e. all their dependencies in the Nix store), and copies
 all paths in the closure to the remote machine via the
 <command>ssh</command> (Secure Shell) command.  With the
 <option>--from</option>, the direction is reversed:
-the closure of <replaceable>paths</replaceable> on a remote machine is
+the closure of <emphasis>paths</emphasis> on a remote machine is
 copied to the Nix store on the local machine.</para>
 
 <para>This command is efficient because it only sends the store paths
@@ -73,8 +73,8 @@ those paths.  If this bothers you, use
   <varlistentry><term><option>--to</option></term>
 
     <listitem><para>Copy the closure of
-    <replaceable>paths</replaceable> from the local Nix store to the
-    Nix store on <replaceable>machine</replaceable>.  This is the
+    <emphasis>paths</emphasis> from the local Nix store to the
+    Nix store on <emphasis>machine</emphasis>.  This is the
     default.</para></listitem>
 
   </varlistentry>
@@ -82,8 +82,8 @@ those paths.  If this bothers you, use
   <varlistentry><term><option>--from</option></term>
 
     <listitem><para>Copy the closure of
-    <replaceable>paths</replaceable> from the Nix store on
-    <replaceable>machine</replaceable> to the local Nix
+    <emphasis>paths</emphasis> from the Nix store on
+    <emphasis>machine</emphasis> to the local Nix
     store.</para></listitem>
 
   </varlistentry>
diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 4dedd90ca..6a97c7e37 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -20,30 +20,30 @@
   <cmdsynopsis>
     <command>nix-env</command>
     <xi:include href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-    <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
-    <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
+    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
+    <arg><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--file</option></arg>
         <arg choice='plain'><option>-f</option></arg>
       </group>
-      <replaceable>path</replaceable>
+      <emphasis>path</emphasis>
     </arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--profile</option></arg>
         <arg choice='plain'><option>-p</option></arg>
       </group>
-      <replaceable>path</replaceable>
+      <emphasis>path</emphasis>
     </arg>
     <arg>
       <arg choice='plain'><option>--system-filter</option></arg>
-      <replaceable>system</replaceable>
+      <emphasis>system</emphasis>
     </arg>
     <arg><option>--dry-run</option></arg>
-    <arg choice='plain'><replaceable>operation</replaceable></arg>
-    <arg rep='repeat'><replaceable>options</replaceable></arg>
-    <arg rep='repeat'><replaceable>arguments</replaceable></arg>
+    <arg choice='plain'><emphasis>operation</emphasis></arg>
+    <arg rep='repeat'><emphasis>options</emphasis></arg>
+    <arg rep='repeat'><emphasis>arguments</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
@@ -146,7 +146,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <variablelist>
 
-  <varlistentry><term><option>--file</option> / <option>-f</option> <replaceable>path</replaceable></term>
+  <varlistentry><term><option>--file</option> / <option>-f</option> <emphasis>path</emphasis></term>
 
     <listitem><para>Specifies the Nix expression (designated below as
     the <emphasis>active Nix expression</emphasis>) used by the
@@ -165,7 +165,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--profile</option> / <option>-p</option> <replaceable>path</replaceable></term>
+  <varlistentry><term><option>--profile</option> / <option>-p</option> <emphasis>path</emphasis></term>
 
     <listitem><para>Specifies the profile to be used by those
     operations that operate on a profile (designated below as the
@@ -194,12 +194,12 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--system-filter</option> <replaceable>system</replaceable></term>
+  <varlistentry><term><option>--system-filter</option> <emphasis>system</emphasis></term>
 
     <listitem><para>By default, operations such as <option>--query
     --available</option> show derivations matching any platform.  This
     option allows you to use derivations for the specified platform
-    <replaceable>system</replaceable>.</para></listitem>
+    <emphasis>system</emphasis>.</para></listitem>
 
   </varlistentry>
 
@@ -276,7 +276,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     <listitem><para>A symbolic link to the user's current profile.  By
     default, this symlink points to
-    <filename><replaceable>prefix</replaceable>/var/nix/profiles/default</filename>.
+    <filename><emphasis>prefix</emphasis>/var/nix/profiles/default</filename>.
     The <literal>PATH</literal> environment variable should include
     <filename>~/.nix-profile/bin</filename> for the user environment
     to be visible to the user.</para></listitem>
@@ -310,7 +310,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
     <arg choice='plain'><option>--remove-all</option></arg>
     <arg choice='plain'><option>-r</option></arg>
   </group>
-  <arg choice='plain' rep='repeat'><replaceable>args</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>args</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -320,13 +320,13 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <para>The install operation creates a new user environment, based on
 the current generation of the active profile, to which a set of store
-paths described by <replaceable>args</replaceable> is added.  The
-arguments <replaceable>args</replaceable> map to store paths in a
+paths described by <emphasis>args</emphasis> is added.  The
+arguments <emphasis>args</emphasis> map to store paths in a
 number of possible ways:
 
 <itemizedlist>
 
-  <listitem><para>By default, <replaceable>args</replaceable> is a set
+  <listitem><para>By default, <emphasis>args</emphasis> is a set
   of derivation names denoting derivations in the active Nix
   expression.  These are realised, and the resulting output paths are
   installed.  Currently installed derivations with a name equal to the
@@ -335,7 +335,7 @@ number of possible ways:
   specified.</para>
 
   <para>If there are multiple derivations matching a name in
-  <replaceable>args</replaceable> that have the same name (e.g.,
+  <emphasis>args</emphasis> that have the same name (e.g.,
   <literal>gcc-3.3.6</literal> and <literal>gcc-4.1.1</literal>), then
   the derivation with the highest <emphasis>priority</emphasis> is
   used.  A derivation can define a priority by declaring the
@@ -362,14 +362,14 @@ number of possible ways:
   packages, use <literal>nix-env -qaP</literal>.</para></listitem>
 
   <listitem><para>If <option>--from-profile</option>
-  <replaceable>path</replaceable> is given,
-  <replaceable>args</replaceable> is a set of names denoting installed
-  store paths in the profile <replaceable>path</replaceable>.  This is
+  <emphasis>path</emphasis> is given,
+  <emphasis>args</emphasis> is a set of names denoting installed
+  store paths in the profile <emphasis>path</emphasis>.  This is
   an easy way to copy user environment elements from one profile to
   another.</para></listitem>
 
   <listitem><para>If <option>--from-expression</option> is given,
-  <replaceable>args</replaceable> are Nix <link
+  <emphasis>args</emphasis> are Nix <link
   linkend="ss-functions">functions</link> that are called with the
   active Nix expression as their single argument.  The derivations
   returned by those function calls are installed.  This allows
@@ -377,12 +377,12 @@ number of possible ways:
   if there are multiple derivations with the same
   name.</para></listitem>
 
-  <listitem><para>If <replaceable>args</replaceable> are store
+  <listitem><para>If <emphasis>args</emphasis> are store
   derivations, then these are <link
   linkend="rsec-nix-store-realise">realised</link>, and the resulting
   output paths are installed.</para></listitem>
 
-  <listitem><para>If <replaceable>args</replaceable> are store paths
+  <listitem><para>If <emphasis>args</emphasis> are store paths
   that are not store derivations, then these are <link
   linkend="rsec-nix-store-realise">realised</link> and
   installed.</para></listitem>
@@ -518,7 +518,7 @@ $ nix-env -f '&lt;nixpkgs>' -iA hello --dry-run
 installing ‘hello-2.10’
 this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
   /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
-  <replaceable>...</replaceable></screen>
+  <emphasis>...</emphasis></screen>
 
 </para>
 
@@ -556,7 +556,7 @@ $ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA fir
     <arg choice='plain'><option>--eq</option></arg>
     <arg choice='plain'><option>--always</option></arg>
   </group>
-  <arg choice='plain' rep='repeat'><replaceable>args</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>args</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -566,15 +566,15 @@ $ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA fir
 <para>The upgrade operation creates a new user environment, based on
 the current generation of the active profile, in which all store paths
 are replaced for which there are newer versions in the set of paths
-described by <replaceable>args</replaceable>.  Paths for which there
+described by <emphasis>args</emphasis>.  Paths for which there
 are no newer versions are left untouched; this is not an error.  It is
-also not an error if an element of <replaceable>args</replaceable>
+also not an error if an element of <emphasis>args</emphasis>
 matches no installed derivations.</para>
 
-<para>For a description of how <replaceable>args</replaceable> is
+<para>For a description of how <emphasis>args</emphasis> is
 mapped to a set of store paths, see <link
 linkend="rsec-nix-env-install"><option>--install</option></link>.  If
-<replaceable>args</replaceable> describes multiple store paths with
+<emphasis>args</emphasis> describes multiple store paths with
 the same symbolic name, only the one with the highest version is
 installed.</para>
 
@@ -711,7 +711,7 @@ lexicographically (i.e., using case-sensitive string comparison).</para>
     <arg choice='plain'><option>--uninstall</option></arg>
     <arg choice='plain'><option>-e</option></arg>
   </group>
-  <arg choice='plain' rep='repeat'><replaceable>drvnames</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>drvnames</emphasis></arg>
 </cmdsynopsis>
 </refsection>
 
@@ -720,7 +720,7 @@ lexicographically (i.e., using case-sensitive string comparison).</para>
 <para>The uninstall operation creates a new user environment, based on
 the current generation of the active profile, from which the store
 paths designated by the symbolic names
-<replaceable>names</replaceable> are removed.</para>
+<emphasis>names</emphasis> are removed.</para>
 
 </refsection>
 
@@ -745,7 +745,7 @@ $ nix-env -e '.*' <lineannotation>(remove everything)</lineannotation></screen>
 <cmdsynopsis>
   <command>nix-env</command>
   <arg choice='plain'><option>--set</option></arg>
-  <arg choice='plain'><replaceable>drvname</replaceable></arg>
+  <arg choice='plain'><emphasis>drvname</emphasis></arg>
 </cmdsynopsis>
 </refsection>
 
@@ -783,9 +783,9 @@ $ nix-env -p /nix/var/nix/profiles/browser --set firefox</screen>
 <cmdsynopsis>
   <command>nix-env</command>
   <arg choice='plain'><option>--set-flag</option></arg>
-  <arg choice='plain'><replaceable>name</replaceable></arg>
-  <arg choice='plain'><replaceable>value</replaceable></arg>
-  <arg choice='plain' rep='repeat'><replaceable>drvnames</replaceable></arg>
+  <arg choice='plain'><emphasis>name</emphasis></arg>
+  <arg choice='plain'><emphasis>value</emphasis></arg>
+  <arg choice='plain' rep='repeat'><emphasis>drvnames</emphasis></arg>
 </cmdsynopsis>
 </refsection>
 
@@ -848,8 +848,8 @@ firefox-2.0.0.9 <lineannotation>(the current one)</lineannotation>
 $ nix-env --preserve-installed -i firefox-2.0.0.11
 installing `firefox-2.0.0.11'
 building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
-collision between `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.11/bin/firefox'
-  and `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.9/bin/firefox'.
+collision between `/nix/store/<emphasis>...</emphasis>-firefox-2.0.0.11/bin/firefox'
+  and `/nix/store/<emphasis>...</emphasis>-firefox-2.0.0.9/bin/firefox'.
 <lineannotation>(i.e., can’t have two active at the same time)</lineannotation>
 
 $ nix-env --set-flag active false firefox
@@ -940,12 +940,12 @@ $ nix-env --set-flag priority 10 gcc</screen>
       <arg choice='plain'><option>--attr</option></arg>
       <arg choice='plain'><option>-A</option></arg>
     </group>
-    <replaceable>attribute-path</replaceable>
+    <emphasis>attribute-path</emphasis>
   </arg>
 
   <sbr />
 
-  <arg choice='plain' rep='repeat'><replaceable>names</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>names</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -959,7 +959,7 @@ profile (<option>--installed</option>), or the derivations that are
 available for installation in the active Nix expression
 (<option>--available</option>).  It only prints information about
 derivations whose symbolic name matches one of
-<replaceable>names</replaceable>.</para>
+<emphasis>names</emphasis>.</para>
 
 <para>The derivations are sorted by their <literal>name</literal>
 attributes.</para>
@@ -1086,21 +1086,21 @@ user environment elements, etc. -->
 
     <variablelist>
 
-      <varlistentry><term><literal>&lt;</literal> <replaceable>version</replaceable></term>
+      <varlistentry><term><literal>&lt;</literal> <emphasis>version</emphasis></term>
 
         <listitem><para>A newer version of the package is available
         or installed.</para></listitem>
 
       </varlistentry>
 
-      <varlistentry><term><literal>=</literal> <replaceable>version</replaceable></term>
+      <varlistentry><term><literal>=</literal> <emphasis>version</emphasis></term>
 
         <listitem><para>At most the same version of the package is
         available or installed.</para></listitem>
 
       </varlistentry>
 
-      <varlistentry><term><literal>></literal> <replaceable>version</replaceable></term>
+      <varlistentry><term><literal>></literal> <emphasis>version</emphasis></term>
 
         <listitem><para>Only older versions of the package are
         available or installed.</para></listitem>
@@ -1174,7 +1174,7 @@ docbook-xml-4.2
 firefox-1.0.4
 MPlayer-1.0pre7
 ORBit2-2.8.3
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 </screen>
 
 </para>
@@ -1187,7 +1187,7 @@ firefox-1.0.7
 GConf-2.4.0.1
 MPlayer-1.0pre7
 ORBit2-2.8.3
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 </screen>
 
 </para>
@@ -1200,7 +1200,7 @@ $ nix-env -qas
 --S GConf-2.4.0.1   <lineannotation>(not present, but there is a substitute for fast installation)</lineannotation>
 --S MPlayer-1.0pre3 <lineannotation>(i.e., this is not the installed MPlayer, even though the version is the same!)</lineannotation>
 IP- ORBit2-2.8.3    <lineannotation>(installed and by definition present)</lineannotation>
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 </screen>
 
 </para>
@@ -1218,11 +1218,11 @@ foo-1.2.3
 
 <screen>
 $ nix-env -qc
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 acrobat-reader-7.0 - ?      <lineannotation>(package is not available at all)</lineannotation>
 autoconf-2.59      = 2.59   <lineannotation>(same version)</lineannotation>
 firefox-1.0.4      &lt; 1.0.7  <lineannotation>(a more recent version is available)</lineannotation>
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 </screen>
 
 </para>
@@ -1234,7 +1234,7 @@ $ nix-env -qa '.*zip.*'
 bzip2-1.0.6
 gzip-1.6
 zip-3.0
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 </screen>
 
 </para>
@@ -1248,7 +1248,7 @@ chromium-37.0.2062.94
 chromium-beta-38.0.2125.24
 firefox-32.0.3
 firefox-with-plugins-13.0.1
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 </screen>
 
 </para>
@@ -1280,7 +1280,7 @@ $ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
     <arg choice='plain'><option>--switch-profile</option></arg>
     <arg choice='plain'><option>-S</option></arg>
   </group>
-  <arg choice='req'><replaceable>path</replaceable></arg>
+  <arg choice='req'><emphasis>path</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -1288,10 +1288,10 @@ $ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
 
 <refsection><title>Description</title>
 
-<para>This operation makes <replaceable>path</replaceable> the current
+<para>This operation makes <emphasis>path</emphasis> the current
 profile for the user.  That is, the symlink
 <filename>~/.nix-profile</filename> is made to point to
-<replaceable>path</replaceable>.</para>
+<emphasis>path</emphasis>.</para>
 
 </refsection>
 
@@ -1355,7 +1355,7 @@ $ nix-env --list-generations
 <cmdsynopsis>
   <command>nix-env</command>
   <arg choice='plain'><option>--delete-generations</option></arg>
-  <arg choice='plain' rep='repeat'><replaceable>generations</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>generations</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -1407,7 +1407,7 @@ $ nix-env -p other_profile --delete-generations old</screen>
     <arg choice='plain'><option>--switch-generation</option></arg>
     <arg choice='plain'><option>-G</option></arg>
   </group>
-  <arg choice='req'><replaceable>generation</replaceable></arg>
+  <arg choice='req'><emphasis>generation</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -1416,13 +1416,13 @@ $ nix-env -p other_profile --delete-generations old</screen>
 <refsection><title>Description</title>
 
 <para>This operation makes generation number
-<replaceable>generation</replaceable> the current generation of the
+<emphasis>generation</emphasis> the current generation of the
 active profile.  That is, if the
-<filename><replaceable>profile</replaceable></filename> is the path to
+<filename><emphasis>profile</emphasis></filename> is the path to
 the active profile, then the symlink
-<filename><replaceable>profile</replaceable></filename> is made to
+<filename><emphasis>profile</emphasis></filename> is made to
 point to
-<filename><replaceable>profile</replaceable>-<replaceable>generation</replaceable>-link</filename>,
+<filename><emphasis>profile</emphasis>-<emphasis>generation</emphasis>-link</filename>,
 which is in turn a symlink to the actual user environment in the Nix
 store.</para>
 
diff --git a/doc/manual/command-ref/nix-hash.xml b/doc/manual/command-ref/nix-hash.xml
index 80263e18e..2f80dc568 100644
--- a/doc/manual/command-ref/nix-hash.xml
+++ b/doc/manual/command-ref/nix-hash.xml
@@ -22,18 +22,18 @@
     <arg><option>--flat</option></arg>
     <arg><option>--base32</option></arg>
     <arg><option>--truncate</option></arg>
-    <arg><option>--type</option> <replaceable>hashAlgo</replaceable></arg>
-    <arg choice='plain' rep='repeat'><replaceable>path</replaceable></arg>
+    <arg><option>--type</option> <emphasis>hashAlgo</emphasis></arg>
+    <arg choice='plain' rep='repeat'><emphasis>path</emphasis></arg>
   </cmdsynopsis>
   <cmdsynopsis>
     <command>nix-hash</command>
     <arg choice='plain'><option>--to-base16</option></arg>
-    <arg choice='plain' rep='repeat'><replaceable>hash</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>hash</emphasis></arg>
   </cmdsynopsis>
   <cmdsynopsis>
     <command>nix-hash</command>
     <arg choice='plain'><option>--to-base32</option></arg>
-    <arg choice='plain' rep='repeat'><replaceable>hash</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>hash</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
@@ -42,7 +42,7 @@
 
 <para>The command <command>nix-hash</command> computes the
 cryptographic hash of the contents of each
-<replaceable>path</replaceable> and prints it on standard output.  By
+<emphasis>path</emphasis> and prints it on standard output.  By
 default, it computes an MD5 hash, but other hash algorithms are
 available as well.  The hash is printed in hexadecimal.  To generate
 the same hash as <command>nix-prefetch-url</command> you have to
@@ -54,9 +54,9 @@ allows directories and symlinks to be hashed as well as regular files.
 The dump is in the <emphasis>NAR format</emphasis> produced by <link
 linkend="refsec-nix-store-dump"><command>nix-store</command>
 <option>--dump</option></link>.  Thus, <literal>nix-hash
-<replaceable>path</replaceable></literal> yields the same
+<emphasis>path</emphasis></literal> yields the same
 cryptographic hash as <literal>nix-store --dump
-<replaceable>path</replaceable> | md5sum</literal>.</para>
+<emphasis>path</emphasis> | md5sum</literal>.</para>
 
 </refsection>
 
@@ -68,9 +68,9 @@ cryptographic hash as <literal>nix-store --dump
   <varlistentry><term><option>--flat</option></term>
 
     <listitem><para>Print the cryptographic hash of the contents of
-    each regular file <replaceable>path</replaceable>.  That is, do
+    each regular file <emphasis>path</emphasis>.  That is, do
     not compute the hash over the dump of
-    <replaceable>path</replaceable>.  The result is identical to that
+    <emphasis>path</emphasis>.  The result is identical to that
     produced by the GNU commands <command>md5sum</command> and
     <command>sha1sum</command>.</para></listitem>
 
@@ -92,7 +92,7 @@ cryptographic hash as <literal>nix-store --dump
 
   </varlistentry>
 
-  <varlistentry><term><option>--type</option> <replaceable>hashAlgo</replaceable></term>
+  <varlistentry><term><option>--type</option> <emphasis>hashAlgo</emphasis></term>
 
     <listitem><para>Use the specified cryptographic hash algorithm,
     which can be one of <literal>md5</literal>,
@@ -104,7 +104,7 @@ cryptographic hash as <literal>nix-store --dump
   <varlistentry><term><option>--to-base16</option></term>
 
     <listitem><para>Don’t hash anything, but convert the base-32 hash
-    representation <replaceable>hash</replaceable> to
+    representation <emphasis>hash</emphasis> to
     hexadecimal.</para></listitem>
 
   </varlistentry>
@@ -112,7 +112,7 @@ cryptographic hash as <literal>nix-store --dump
   <varlistentry><term><option>--to-base32</option></term>
 
     <listitem><para>Don’t hash anything, but convert the hexadecimal
-    hash representation <replaceable>hash</replaceable> to
+    hash representation <emphasis>hash</emphasis> to
     base-32.</para></listitem>
 
   </varlistentry>
diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index 4ff967ecf..d67be4924 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -29,26 +29,26 @@
       </arg>
     </group>
     <arg><option>--read-write-mode</option></arg>
-    <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
+    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--attr</option></arg>
         <arg choice='plain'><option>-A</option></arg>
       </group>
-      <replaceable>attrPath</replaceable>
+      <emphasis>attrPath</emphasis>
     </arg>
-    <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
+    <arg><option>--add-root</option> <emphasis>path</emphasis></arg>
     <arg><option>--indirect</option></arg>
     <group>
       <arg choice='plain'><option>--expr</option></arg>
       <arg choice='plain'><option>-E</option></arg>
     </group>
-    <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>files</emphasis></arg>
   </cmdsynopsis>
   <cmdsynopsis>
     <command>nix-instantiate</command>
     <arg choice='plain'><option>--find-file</option></arg>
-    <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>files</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
@@ -58,13 +58,13 @@
 <para>The command <command>nix-instantiate</command> generates <link
 linkend="gloss-derivation">store derivations</link> from (high-level)
 Nix expressions.  It evaluates the Nix expressions in each of
-<replaceable>files</replaceable> (which defaults to
-<replaceable>./default.nix</replaceable>).  Each top-level expression
+<emphasis>files</emphasis> (which defaults to
+<emphasis>./default.nix</emphasis>).  Each top-level expression
 should evaluate to a derivation, a list of derivations, or a set of
 derivations.  The paths of the resulting store derivations are printed
 on standard output.</para>
 
-<para>If <replaceable>files</replaceable> is the character
+<para>If <emphasis>files</emphasis> is the character
 <literal>-</literal>, then a Nix expression will be read from standard
 input.</para>
 
@@ -79,7 +79,7 @@ input.</para>
 <variablelist>
 
   <varlistentry>
-    <term><option>--add-root</option> <replaceable>path</replaceable></term>
+    <term><option>--add-root</option> <emphasis>path</emphasis></term>
     <term><option>--indirect</option></term>
 
     <listitem><para>See the <link linkend="opt-add-root">corresponding
@@ -181,7 +181,7 @@ $ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
 /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
 
 $ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
 
 $ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
@@ -226,28 +226,28 @@ $ nix-instantiate --eval --xml -E '1 + 2'
 
 <screen>
 $ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }'
-<replaceable>...</replaceable><![CDATA[
+<emphasis>...</emphasis><![CDATA[
   <attr name="x">
     <string value="foo" />
   </attr>
   <attr name="y">
     <unevaluated />
   </attr>]]>
-<replaceable>...</replaceable></screen>
+<emphasis>...</emphasis></screen>
 
 Note that <varname>y</varname> is left unevaluated (the XML
 representation doesn’t attempt to show non-normal forms).
 
 <screen>
 $ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }'
-<replaceable>...</replaceable><![CDATA[
+<emphasis>...</emphasis><![CDATA[
   <attr name="x">
     <string value="foo" />
   </attr>
   <attr name="y">
     <string value="foo" />
   </attr>]]>
-<replaceable>...</replaceable></screen>
+<emphasis>...</emphasis></screen>
 
 </para>
 
diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml
index 621ded72e..db2a6960a 100644
--- a/doc/manual/command-ref/nix-prefetch-url.xml
+++ b/doc/manual/command-ref/nix-prefetch-url.xml
@@ -20,24 +20,24 @@
   <cmdsynopsis>
     <command>nix-prefetch-url</command>
     <arg><option>--version</option></arg>
-    <arg><option>--type</option> <replaceable>hashAlgo</replaceable></arg>
+    <arg><option>--type</option> <emphasis>hashAlgo</emphasis></arg>
     <arg><option>--print-path</option></arg>
     <arg><option>--unpack</option></arg>
-    <arg><option>--name</option> <replaceable>name</replaceable></arg>
-    <arg choice='plain'><replaceable>url</replaceable></arg>
-    <arg><replaceable>hash</replaceable></arg>
+    <arg><option>--name</option> <emphasis>name</emphasis></arg>
+    <arg choice='plain'><emphasis>url</emphasis></arg>
+    <arg><emphasis>hash</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
 <refsection><title>Description</title>
 
 <para>The command <command>nix-prefetch-url</command> downloads the
-file referenced by the URL <replaceable>url</replaceable>, prints its
+file referenced by the URL <emphasis>url</emphasis>, prints its
 cryptographic hash, and copies it into the Nix store.  The file name
 in the store is
-<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>,
-where <replaceable>baseName</replaceable> is everything following the
-final slash in <replaceable>url</replaceable>.</para>
+<filename><emphasis>hash</emphasis>-<emphasis>baseName</emphasis></filename>,
+where <emphasis>baseName</emphasis> is everything following the
+final slash in <emphasis>url</emphasis>.</para>
 
 <para>This command is just a convenience for Nix expression writers.
 Often a Nix expression fetches some source distribution from the
@@ -50,7 +50,7 @@ download it again when you build your Nix expression.  Since
 as <command>nix-prefetch-url</command>, the redundant download can be
 avoided.</para>
 
-<para>If <replaceable>hash</replaceable> is specified, then a download
+<para>If <emphasis>hash</emphasis> is specified, then a download
 is not performed if the Nix store already contains a file with the
 same hash and base name.  Otherwise, the file is downloaded, and an
 error is signaled if the actual hash of the file does not match the
@@ -67,7 +67,7 @@ downloaded file in the Nix store is also printed.</para>
 
 <variablelist>
 
-  <varlistentry><term><option>--type</option> <replaceable>hashAlgo</replaceable></term>
+  <varlistentry><term><option>--type</option> <emphasis>hashAlgo</emphasis></term>
 
     <listitem><para>Use the specified cryptographic hash algorithm,
     which can be one of <literal>md5</literal>,
@@ -93,14 +93,14 @@ downloaded file in the Nix store is also printed.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--name</option> <replaceable>name</replaceable></term>
+  <varlistentry><term><option>--name</option> <emphasis>name</emphasis></term>
 
     <listitem><para>Override the name of the file in the Nix store. By
     default, this is
-    <literal><replaceable>hash</replaceable>-<replaceable>basename</replaceable></literal>,
-    where <replaceable>basename</replaceable> is the last component of
-    <replaceable>url</replaceable>. Overriding the name is necessary
-    when <replaceable>basename</replaceable> contains characters that
+    <literal><emphasis>hash</emphasis>-<emphasis>basename</emphasis></literal>,
+    where <emphasis>basename</emphasis> is the last component of
+    <emphasis>url</emphasis>. Overriding the name is necessary
+    when <emphasis>basename</emphasis> contains characters that
     are not allowed in Nix store paths.</para></listitem>
 
   </varlistentry>
diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index de9755d58..1d55b5622 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -19,20 +19,20 @@
 <refsynopsisdiv>
   <cmdsynopsis>
     <command>nix-shell</command>
-    <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
-    <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
+    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
+    <arg><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--attr</option></arg>
         <arg choice='plain'><option>-A</option></arg>
       </group>
-      <replaceable>attrPath</replaceable>
+      <emphasis>attrPath</emphasis>
     </arg>
-    <arg><option>--command</option> <replaceable>cmd</replaceable></arg>
-    <arg><option>--run</option> <replaceable>cmd</replaceable></arg>
-    <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
+    <arg><option>--command</option> <emphasis>cmd</emphasis></arg>
+    <arg><option>--run</option> <emphasis>cmd</emphasis></arg>
+    <arg><option>--exclude</option> <emphasis>regexp</emphasis></arg>
     <arg><option>--pure</option></arg>
-    <arg><option>--keep</option> <replaceable>name</replaceable></arg>
+    <arg><option>--keep</option> <emphasis>name</emphasis></arg>
     <group choice='req'>
       <arg choice='plain'>
         <group choice='req'>
@@ -41,12 +41,12 @@
         </group>
         <arg choice='plain' rep='repeat'>
           <group choice='req'>
-            <arg choice="plain"><replaceable>packages</replaceable></arg>
-            <arg choice="plain"><replaceable>expressions</replaceable></arg>
+            <arg choice="plain"><emphasis>packages</emphasis></arg>
+            <arg choice="plain"><emphasis>expressions</emphasis></arg>
           </group>
         </arg>
       </arg>
-      <arg><replaceable>path</replaceable></arg>
+      <arg><emphasis>path</emphasis></arg>
     </group>
   </cmdsynopsis>
 </refsynopsisdiv>
@@ -57,17 +57,17 @@
 dependencies of the specified derivation, but not the derivation
 itself.  It will then start an interactive shell in which all
 environment variables defined by the derivation
-<replaceable>path</replaceable> have been set to their corresponding
+<emphasis>path</emphasis> have been set to their corresponding
 values, and the script <literal>$stdenv/setup</literal> has been
 sourced.  This is useful for reproducing the environment of a
 derivation for development.</para>
 
-<para>If <replaceable>path</replaceable> is not given,
+<para>If <emphasis>path</emphasis> is not given,
 <command>nix-shell</command> defaults to
 <filename>shell.nix</filename> if it exists, and
 <filename>default.nix</filename> otherwise.</para>
 
-<para>If <replaceable>path</replaceable> starts with
+<para>If <emphasis>path</emphasis> starts with
 <literal>http://</literal> or <literal>https://</literal>, it is
 interpreted as the URL of a tarball that will be downloaded and
 unpacked to a temporary location. The tarball must include a single
@@ -103,10 +103,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <variablelist>
 
-  <varlistentry><term><option>--command</option> <replaceable>cmd</replaceable></term>
+  <varlistentry><term><option>--command</option> <emphasis>cmd</emphasis></term>
 
     <listitem><para>In the environment of the derivation, run the
-    shell command <replaceable>cmd</replaceable>. This command is
+    shell command <emphasis>cmd</emphasis>. This command is
     executed in an interactive shell. (Use <option>--run</option> to
     use a non-interactive shell instead.) However, a call to
     <literal>exit</literal> is implicitly added to the command, so the
@@ -118,7 +118,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--run</option> <replaceable>cmd</replaceable></term>
+  <varlistentry><term><option>--run</option> <emphasis>cmd</emphasis></term>
 
     <listitem><para>Like <option>--command</option>, but executes the
     command in a non-interactive shell. This means (among other
@@ -127,10 +127,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--exclude</option> <replaceable>regexp</replaceable></term>
+  <varlistentry><term><option>--exclude</option> <emphasis>regexp</emphasis></term>
 
     <listitem><para>Do not build any dependencies whose store path
-    matches the regular expression <replaceable>regexp</replaceable>.
+    matches the regular expression <emphasis>regexp</emphasis>.
     This option may be specified multiple times.</para></listitem>
 
   </varlistentry>
@@ -150,7 +150,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--packages</option> / <option>-p</option> <replaceable>packages</replaceable>…</term>
+  <varlistentry><term><option>--packages</option> / <option>-p</option> <emphasis>packages</emphasis>…</term>
 
     <listitem><para>Set up an environment in which the specified
     packages are present.  The command line arguments are interpreted
@@ -162,7 +162,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>-i</option> <replaceable>interpreter</replaceable></term>
+  <varlistentry><term><option>-i</option> <emphasis>interpreter</emphasis></term>
 
     <listitem><para>The chained script interpreter to be invoked by
     <command>nix-shell</command>. Only applicable in
@@ -171,7 +171,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     </listitem></varlistentry>
 
-  <varlistentry><term><option>--keep</option> <replaceable>name</replaceable></term>
+  <varlistentry><term><option>--keep</option> <emphasis>name</emphasis></term>
 
     <listitem><para>When a <option>--pure</option> shell is started,
     keep the listed environment variables.</para></listitem>
@@ -278,13 +278,13 @@ following lines:
 
 <programlisting>
 #! /usr/bin/env nix-shell
-#! nix-shell -i <replaceable>real-interpreter</replaceable> -p <replaceable>packages</replaceable>
+#! nix-shell -i <emphasis>real-interpreter</emphasis> -p <emphasis>packages</emphasis>
 </programlisting>
 
-where <replaceable>real-interpreter</replaceable> is the “real” script
+where <emphasis>real-interpreter</emphasis> is the “real” script
 interpreter that will be invoked by <command>nix-shell</command> after
 it has obtained the dependencies and initialised the environment, and
-<replaceable>packages</replaceable> are the attribute names of the
+<emphasis>packages</emphasis> are the attribute names of the
 dependencies in Nixpkgs.</para>
 
 <para>The lines starting with <literal>#! nix-shell</literal> specify
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 459100984..352e716ae 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -20,11 +20,11 @@
   <cmdsynopsis>
     <command>nix-store</command>
     <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-    <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
+    <arg><option>--add-root</option> <emphasis>path</emphasis></arg>
     <arg><option>--indirect</option></arg>
-    <arg choice='plain'><replaceable>operation</replaceable></arg>
-    <arg rep='repeat'><replaceable>options</replaceable></arg>
-    <arg rep='repeat'><replaceable>arguments</replaceable></arg>
+    <arg choice='plain'><emphasis>operation</emphasis></arg>
+    <arg rep='repeat'><emphasis>options</emphasis></arg>
+    <arg rep='repeat'><emphasis>arguments</emphasis></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
@@ -55,14 +55,14 @@ options.</phrase></para>
 
 <variablelist>
 
-  <varlistentry xml:id="opt-add-root"><term><option>--add-root</option> <replaceable>path</replaceable></term>
+  <varlistentry xml:id="opt-add-root"><term><option>--add-root</option> <emphasis>path</emphasis></term>
 
     <listitem><para>Causes the result of a realisation
     (<option>--realise</option> and <option>--force-realise</option>)
     to be registered as a root of the garbage collector<phrase
     condition="manual"> (see <xref linkend="ssec-gc-roots"
     />)</phrase>.  The root is stored in
-    <replaceable>path</replaceable>, which must be inside a directory
+    <emphasis>path</emphasis>, which must be inside a directory
     that is scanned for roots by the garbage collector (i.e.,
     typically in a subdirectory of
     <filename>/nix/var/nix/gcroots/</filename>)
@@ -88,11 +88,11 @@ options.</phrase></para>
     garbage-collected unless the symlink is removed.</para>
 
     <para>The <option>--indirect</option> flag causes a uniquely named
-    symlink to <replaceable>path</replaceable> to be stored in
+    symlink to <emphasis>path</emphasis> to be stored in
     <filename>/nix/var/nix/gcroots/auto/</filename>.  For instance,
 
     <screen>
-$ nix-store --add-root /home/eelco/bla/result --indirect -r <replaceable>...</replaceable>
+$ nix-store --add-root /home/eelco/bla/result --indirect -r <emphasis>...</emphasis>
 
 $ ls -l /nix/var/nix/gcroots/auto
 lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
@@ -135,7 +135,7 @@ lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134
     <arg choice='plain'><option>--realise</option></arg>
     <arg choice='plain'><option>-r</option></arg>
   </group>
-  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
   <arg><option>--dry-run</option></arg>
 </cmdsynopsis>
 
@@ -204,7 +204,7 @@ printed.)</para>
     with <option>-K</option>, if an output path is not identical to
     the corresponding output from the previous build, the new output
     path is left in
-    <filename>/nix/store/<replaceable>name</replaceable>.check.</filename></para>
+    <filename>/nix/store/<emphasis>name</emphasis>.check.</filename></para>
 
     <para>See also the <option>build-repeat</option> configuration
     option, which repeats a derivation a number of times and prevents
@@ -361,7 +361,7 @@ EOF
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
   </group>
-  <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
+  <arg><option>--max-freed</option> <emphasis>bytes</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -413,11 +413,11 @@ options control what gets deleted and in what order:
 
 <variablelist>
 
-  <varlistentry><term><option>--max-freed</option> <replaceable>bytes</replaceable></term>
+  <varlistentry><term><option>--max-freed</option> <emphasis>bytes</emphasis></term>
 
     <listitem><para>Keep deleting paths until at least
-    <replaceable>bytes</replaceable> bytes have been deleted, then
-    stop.  The argument <replaceable>bytes</replaceable> can be
+    <emphasis>bytes</emphasis> bytes have been deleted, then
+    stop.  The argument <emphasis>bytes</emphasis> can be
     followed by the multiplicative suffix <literal>K</literal>,
     <literal>M</literal>, <literal>G</literal> or
     <literal>T</literal>, denoting KiB, MiB, GiB or TiB
@@ -450,7 +450,7 @@ be freed.</para>
 <screen>
 $ nix-store --gc
 deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 8825586 bytes freed (8.42 MiB)</screen>
 
 </para>
@@ -479,7 +479,7 @@ $ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
   <command>nix-store</command>
   <arg choice='plain'><option>--delete</option></arg>
   <arg><option>--ignore-liveness</option></arg>
-  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -487,7 +487,7 @@ $ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
 <refsection><title>Description</title>
 
 <para>The operation <option>--delete</option> deletes the store paths
-<replaceable>paths</replaceable> from the Nix store, but only if it is
+<emphasis>paths</emphasis> from the Nix store, but only if it is
 safe to do so; that is, when the path is not reachable from a root of
 the garbage collector.  This means that you can only delete paths that
 would also be deleted by <literal>nix-store --gc</literal>.  Thus,
@@ -537,8 +537,8 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>-d</option></arg>
     <arg choice='plain'><option>--graph</option></arg>
     <arg choice='plain'><option>--tree</option></arg>
-    <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
-    <arg choice='plain'><option>-b</option> <replaceable>name</replaceable></arg>
+    <arg choice='plain'><option>--binding</option> <emphasis>name</emphasis></arg>
+    <arg choice='plain'><option>-b</option> <emphasis>name</emphasis></arg>
     <arg choice='plain'><option>--hash</option></arg>
     <arg choice='plain'><option>--size</option></arg>
     <arg choice='plain'><option>--roots</option></arg>
@@ -547,7 +547,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
   <arg><option>-u</option></arg>
   <arg><option>--force-realise</option></arg>
   <arg><option>-f</option></arg>
-  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -560,7 +560,7 @@ information about the store paths .  The queries are described below.  At
 most one query can be specified.  The default query is
 <option>--outputs</option>.</para>
 
-<para>The paths <replaceable>paths</replaceable> may also be symlinks
+<para>The paths <emphasis>paths</emphasis> may also be symlinks
 from outside of the Nix store, to the Nix store.  In that case, the
 query is applied to the target of the symlink.</para>
 
@@ -603,7 +603,7 @@ query is applied to the target of the symlink.</para>
 
     <listitem><para>Prints out the <link
     linkend="gloss-output-path">output paths</link> of the store
-    derivations <replaceable>paths</replaceable>.  These are the paths
+    derivations <emphasis>paths</emphasis>.  These are the paths
     that will be produced when the derivation is
     built.</para></listitem>
 
@@ -614,7 +614,7 @@ query is applied to the target of the symlink.</para>
 
     <listitem><para>Prints out the <link
     linkend="gloss-closure">closure</link> of the store path
-    <replaceable>paths</replaceable>.</para>
+    <emphasis>paths</emphasis>.</para>
 
     <para>This query has one option:</para>
 
@@ -647,7 +647,7 @@ query is applied to the target of the symlink.</para>
 
     <listitem><para>Prints the set of <link
     linkend="gloss-reference">references</link> of the store paths
-    <replaceable>paths</replaceable>, that is, their immediate
+    <emphasis>paths</emphasis>, that is, their immediate
     dependencies.  (For <emphasis>all</emphasis> dependencies, use
     <option>--requisites</option>.)</para></listitem>
 
@@ -656,9 +656,9 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--referrers</option></term>
 
     <listitem><para>Prints the set of <emphasis>referrers</emphasis> of
-    the store paths <replaceable>paths</replaceable>, that is, the
+    the store paths <emphasis>paths</emphasis>, that is, the
     store paths currently existing in the Nix store that refer to one
-    of <replaceable>paths</replaceable>.  Note that contrary to the
+    of <emphasis>paths</emphasis>.  Note that contrary to the
     references, the set of referrers is not constant; it can change as
     store paths are added or removed.</para></listitem>
 
@@ -667,11 +667,11 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--referrers-closure</option></term>
 
     <listitem><para>Prints the closure of the set of store paths
-    <replaceable>paths</replaceable> under the referrers relation; that
+    <emphasis>paths</emphasis> under the referrers relation; that
     is, all store paths that directly or indirectly refer to one of
-    <replaceable>paths</replaceable>.  These are all the path currently
+    <emphasis>paths</emphasis>.  These are all the path currently
     in the Nix store that are dependent on
-    <replaceable>paths</replaceable>.</para></listitem>
+    <emphasis>paths</emphasis>.</para></listitem>
 
   </varlistentry>
 
@@ -680,7 +680,7 @@ query is applied to the target of the symlink.</para>
 
     <listitem><para>Prints the <link
     linkend="gloss-deriver">deriver</link> of the store paths
-    <replaceable>paths</replaceable>.  If the path has no deriver
+    <emphasis>paths</emphasis>.  If the path has no deriver
     (e.g., if it is a source file), or if the deriver is not known
     (e.g., in the case of a binary-only deployment), the string
     <literal>unknown-deriver</literal> is printed.</para></listitem>
@@ -690,7 +690,7 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--graph</option></term>
 
     <listitem><para>Prints the references graph of the store paths
-    <replaceable>paths</replaceable> in the format of the
+    <emphasis>paths</emphasis> in the format of the
     <command>dot</command> tool of AT&amp;T's <link
     xlink:href="http://www.graphviz.org/">Graphviz package</link>.
     This can be used to visualise dependency graphs.  To obtain a
@@ -703,7 +703,7 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--tree</option></term>
 
     <listitem><para>Prints the references graph of the store paths
-    <replaceable>paths</replaceable> as a nested ASCII tree.
+    <emphasis>paths</emphasis> as a nested ASCII tree.
     References are ordered by descending closure size; this tends to
     flatten the tree, making it more readable.  The query only
     recurses into a store path when it is first encountered; this
@@ -715,7 +715,7 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--graphml</option></term>
 
     <listitem><para>Prints the references graph of the store paths
-    <replaceable>paths</replaceable> in the <link
+    <emphasis>paths</emphasis> in the <link
     xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format.
     This can be used to visualise dependency graphs. To obtain a
     build-time dependency graph, apply this to a store derivation. To
@@ -724,12 +724,12 @@ query is applied to the target of the symlink.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term>
-    <term><option>-b</option> <replaceable>name</replaceable></term>
+  <varlistentry><term><option>--binding</option> <emphasis>name</emphasis></term>
+    <term><option>-b</option> <emphasis>name</emphasis></term>
 
     <listitem><para>Prints the value of the attribute
-    <replaceable>name</replaceable> (i.e., environment variable) of
-    the store derivations <replaceable>paths</replaceable>.  It is an
+    <emphasis>name</emphasis> (i.e., environment variable) of
+    the store derivations <emphasis>paths</emphasis>.  It is an
     error for a derivation to not have the specified
     attribute.</para></listitem>
 
@@ -738,7 +738,7 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--hash</option></term>
 
     <listitem><para>Prints the SHA-256 hash of the contents of the
-    store paths <replaceable>paths</replaceable> (that is, the hash of
+    store paths <emphasis>paths</emphasis> (that is, the hash of
     the output of <command>nix-store --dump</command> on the given
     paths).  Since the hash is stored in the Nix database, this is a
     fast operation.</para></listitem>
@@ -748,7 +748,7 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--size</option></term>
 
     <listitem><para>Prints the size in bytes of the contents of the
-    store paths <replaceable>paths</replaceable> — to be precise, the
+    store paths <emphasis>paths</emphasis> — to be precise, the
     size of the output of <command>nix-store --dump</command> on the
     given paths.  Note that the actual disk space required by the
     store paths may be higher, especially on filesystems with large
@@ -760,7 +760,7 @@ query is applied to the target of the symlink.</para>
 
     <listitem><para>Prints the garbage collector roots that point,
     directly or indirectly, at the store paths
-    <replaceable>paths</replaceable>.</para></listitem>
+    <emphasis>paths</emphasis>.</para></listitem>
 
   </varlistentry>
 
@@ -778,7 +778,7 @@ query is applied to the target of the symlink.</para>
 $ nix-store -qR $(which svn)
 /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
 /nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
-<replaceable>...</replaceable></screen>
+<emphasis>...</emphasis></screen>
 
 </para>
 
@@ -789,7 +789,7 @@ $ nix-store -qR $(nix-store -qd $(which svn))
 /nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
 /nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
 /nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
-<replaceable>... lots of other paths ...</replaceable></screen>
+<emphasis>... lots of other paths ...</emphasis></screen>
 
 The difference with the previous example is that we ask the closure of
 the derivation (<option>-qd</option>), not the closure of the output
@@ -804,7 +804,7 @@ $ nix-store -q --tree $(nix-store -qd $(which svn))
 +---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
 |   +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
 |   +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
-<replaceable>...</replaceable></screen>
+<emphasis>...</emphasis></screen>
 
 </para>
 
@@ -827,7 +827,7 @@ $ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(wh
 $ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
 /nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
 /nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
-<replaceable>...</replaceable></screen>
+<emphasis>...</emphasis></screen>
 
 Note that <command>ldd</command> is a command that prints out the
 dynamic libraries used by an ELF executable.</para>
@@ -893,7 +893,7 @@ $ nix-store -q --roots $(which svn)
 <cmdsynopsis>
   <command>nix-store</command>
   <arg choice='plain'><option>--add</option></arg>
-  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -926,8 +926,8 @@ $ nix-store --add ./foo.c
   <command>nix-store</command>
   <arg><option>--recursive</option></arg>
   <arg choice='plain'><option>--add-fixed</option></arg>
-  <arg choice='plain'><replaceable>algorithm</replaceable></arg>
-  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg choice='plain'><emphasis>algorithm</emphasis></arg>
+  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -1039,7 +1039,7 @@ in Nix itself.</para>
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--verify-path</option></arg>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1077,7 +1077,7 @@ $ nix-store --verify-path $(nix-store -qR $(which svn))
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--repair-path</option></arg>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1123,7 +1123,7 @@ fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--dump</option></arg>
-    <arg choice='plain'><replaceable>path</replaceable></arg>
+    <arg choice='plain'><emphasis>path</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1131,7 +1131,7 @@ fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
 
 <para>The operation <option>--dump</option> produces a NAR (Nix
 ARchive) file containing the contents of the file system tree rooted
-at <replaceable>path</replaceable>.  The archive is written to
+at <emphasis>path</emphasis>.  The archive is written to
 standard output.</para>
 
 <para>A NAR archive is like a TAR or Zip archive, but it contains only
@@ -1173,14 +1173,14 @@ links, but not other types of files (such as device nodes).</para>
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--restore</option></arg>
-    <arg choice='plain'><replaceable>path</replaceable></arg>
+    <arg choice='plain'><emphasis>path</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
 <refsection><title>Description</title>
 
 <para>The operation <option>--restore</option> unpacks a NAR archive
-to <replaceable>path</replaceable>, which must not already exist.  The
+to <emphasis>path</emphasis>, which must not already exist.  The
 archive is read from standard input.</para>
 
 </refsection>
@@ -1198,7 +1198,7 @@ archive is read from standard input.</para>
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--export</option></arg>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1220,7 +1220,7 @@ that are missing in the target Nix store, the import will fail.  To
 copy a whole closure, do something like:
 
 <screen>
-$ nix-store --export $(nix-store -qR <replaceable>paths</replaceable>) > out</screen>
+$ nix-store --export $(nix-store -qR <emphasis>paths</emphasis>) > out</screen>
 
 To import the whole closure again, run:
 
@@ -1299,7 +1299,7 @@ progress indication.</para>
 <screen>
 $ nix-store --optimise
 hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1'
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 541838819 bytes (516.74 MiB) freed by hard-linking 54143 files;
 there are 114486 files with equal contents out of 215894 files in total
 </screen>
@@ -1322,7 +1322,7 @@ there are 114486 files with equal contents out of 215894 files in total
       <arg choice='plain'><option>--read-log</option></arg>
       <arg choice='plain'><option>-l</option></arg>
     </group>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1351,7 +1351,7 @@ unpacking sources
 unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz
 ktorrent-2.2.1/
 ktorrent-2.2.1/NEWS
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 </screen>
 
 </refsection>
@@ -1369,7 +1369,7 @@ ktorrent-2.2.1/NEWS
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--dump-db</option></arg>
-    <arg rep='repeat'><replaceable>paths</replaceable></arg>
+    <arg rep='repeat'><emphasis>paths</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1424,7 +1424,7 @@ loads it into the Nix database.</para>
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--print-env</option></arg>
-    <arg choice='plain'><replaceable>drvpath</replaceable></arg>
+    <arg choice='plain'><emphasis>drvpath</emphasis></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1441,7 +1441,7 @@ variable <literal>_args</literal>.</para>
 
 <screen>
 $ nix-store --print-env $(nix-instantiate '&lt;nixpkgs>' -A firefox)
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2'
 export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv'
 export system; system='x86_64-linux'
diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml
index 2660e3bb1..6b7b5c833 100644
--- a/doc/manual/command-ref/opt-common-syn.xml
+++ b/doc/manual/command-ref/opt-common-syn.xml
@@ -13,7 +13,7 @@
 </arg>
 <arg>
   <option>--log-format</option>
-  <replaceable>format</replaceable>
+  <emphasis>format</emphasis>
 </arg>
 <arg>
   <group choice='plain'>
@@ -26,19 +26,19 @@
     <arg choice='plain'><option>--max-jobs</option></arg>
     <arg choice='plain'><option>-j</option></arg>
   </group>
-  <replaceable>number</replaceable>
+  <emphasis>number</emphasis>
 </arg>
 <arg>
   <option>--cores</option>
-  <replaceable>number</replaceable>
+  <emphasis>number</emphasis>
 </arg>
 <arg>
   <option>--max-silent-time</option>
-  <replaceable>number</replaceable>
+  <emphasis>number</emphasis>
 </arg>
 <arg>
   <option>--timeout</option>
-  <replaceable>number</replaceable>
+  <emphasis>number</emphasis>
 </arg>
 <arg>
   <group choice='plain'>
@@ -56,12 +56,12 @@
 <arg><option>--readonly-mode</option></arg>
 <arg>
   <option>-I</option>
-  <replaceable>path</replaceable>
+  <emphasis>path</emphasis>
 </arg>
 <arg>
   <option>--option</option>
-  <replaceable>name</replaceable>
-  <replaceable>value</replaceable>
+  <emphasis>name</emphasis>
+  <emphasis>value</emphasis>
 </arg>
 <sbr />
 
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 150e732ff..67950e94b 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -92,12 +92,12 @@
 </varlistentry>
 
 
-<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
+<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <emphasis>format</emphasis></term>
 
   <listitem>
 
   <para>This option can be used to change the output of the log format, with
-  <replaceable>format</replaceable> being one of:</para>
+  <emphasis>format</emphasis> being one of:</para>
 
   <variablelist>
 
@@ -130,13 +130,13 @@
   error.  This option suppresses this behaviour.  Note that the
   builder's standard output and error are always written to a log file
   in
-  <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem>
+  <filename><emphasis>prefix</emphasis>/nix/var/log/nix</filename>.</para></listitem>
 
 </varlistentry>
 
 
 <varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
-<replaceable>number</replaceable></term>
+<emphasis>number</emphasis></term>
 
   <listitem>
 
@@ -166,7 +166,7 @@
   of parallelism.  For instance, in Nixpkgs, if the derivation
   attribute <varname>enableParallelBuilding</varname> is set to
   <literal>true</literal>, the builder passes the
-  <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+  <option>-j<emphasis>N</emphasis></option> flag to GNU Make.
   It defaults to the value of the <link
   linkend='conf-cores'><literal>cores</literal></link>
   configuration setting, if set, or <literal>1</literal> otherwise.
@@ -271,7 +271,7 @@
 </varlistentry>
 
 
-<varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term>
+<varlistentry><term><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></term>
 
   <listitem><para>This option is accepted by
   <command>nix-env</command>, <command>nix-instantiate</command>,
@@ -280,14 +280,14 @@
   automatically try to call functions that
   it encounters.  It can automatically call functions for which every
   argument has a <link linkend='ss-functions'>default value</link>
-  (e.g., <literal>{ <replaceable>argName</replaceable> ?
-  <replaceable>defaultValue</replaceable> }:
-  <replaceable>...</replaceable></literal>).  With
+  (e.g., <literal>{ <emphasis>argName</emphasis> ?
+  <emphasis>defaultValue</emphasis> }:
+  <emphasis>...</emphasis></literal>).  With
   <option>--arg</option>, you can also call functions that have
   arguments without a default value (or override a default value).
   That is, if the evaluator encounters a function with an argument
-  named <replaceable>name</replaceable>, it will call it with value
-  <replaceable>value</replaceable>.</para>
+  named <emphasis>name</emphasis>, it will call it with value
+  <emphasis>value</emphasis>.</para>
 
   <para>For instance, the top-level <literal>default.nix</literal> in
   Nixpkgs is actually a function:
@@ -295,23 +295,23 @@
 <programlisting>
 { # The system (e.g., `i686-linux') for which to build the packages.
   system ? builtins.currentSystem
-  <replaceable>...</replaceable>
-}: <replaceable>...</replaceable></programlisting>
+  <emphasis>...</emphasis>
+}: <emphasis>...</emphasis></programlisting>
 
   So if you call this Nix expression (e.g., when you do
-  <literal>nix-env -i <replaceable>pkgname</replaceable></literal>),
+  <literal>nix-env -i <emphasis>pkgname</emphasis></literal>),
   the function will be called automatically using the value <link
   linkend='builtin-currentSystem'><literal>builtins.currentSystem</literal></link>
   for the <literal>system</literal> argument.  You can override this
   using <option>--arg</option>, e.g., <literal>nix-env -i
-  <replaceable>pkgname</replaceable> --arg system
+  <emphasis>pkgname</emphasis> --arg system
   \"i686-freebsd\"</literal>.  (Note that since the argument is a Nix
   string literal, you have to escape the quotes.)</para></listitem>
 
 </varlistentry>
 
 
-<varlistentry><term><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term>
+<varlistentry><term><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></term>
 
   <listitem><para>This option is like <option>--arg</option>, only the
   value is not a Nix expression but a string.  So instead of
@@ -323,17 +323,17 @@
 
 
 <varlistentry xml:id="opt-attr"><term><option>--attr</option> / <option>-A</option>
-<replaceable>attrPath</replaceable></term>
+<emphasis>attrPath</emphasis></term>
 
   <listitem><para>Select an attribute from the top-level Nix
   expression being evaluated.  (<command>nix-env</command>,
   <command>nix-instantiate</command>, <command>nix-build</command> and
   <command>nix-shell</command> only.)  The <emphasis>attribute
-  path</emphasis> <replaceable>attrPath</replaceable> is a sequence of
+  path</emphasis> <emphasis>attrPath</emphasis> is a sequence of
   attribute names separated by dots.  For instance, given a top-level
-  Nix expression <replaceable>e</replaceable>, the attribute path
+  Nix expression <emphasis>e</emphasis>, the attribute path
   <literal>xorg.xorgserver</literal> would cause the expression
-  <literal><replaceable>e</replaceable>.xorg.xorgserver</literal> to
+  <literal><emphasis>e</emphasis>.xorg.xorgserver</literal> to
   be used.  See <link
   linkend='refsec-nix-env-install-examples'><command>nix-env
   --install</command></link> for some concrete examples.</para>
@@ -366,7 +366,7 @@
 </varlistentry>
 
 
-<varlistentry xml:id="opt-I"><term><option>-I</option> <replaceable>path</replaceable></term>
+<varlistentry xml:id="opt-I"><term><option>-I</option> <emphasis>path</emphasis></term>
 
   <listitem><para>Add a path to the Nix expression search path.  This
   option may be given multiple times.  See the <envar
@@ -378,10 +378,10 @@
 </varlistentry>
 
 
-<varlistentry><term><option>--option</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term>
+<varlistentry><term><option>--option</option> <emphasis>name</emphasis> <emphasis>value</emphasis></term>
 
   <listitem><para>Set the Nix configuration option
-  <replaceable>name</replaceable> to <replaceable>value</replaceable>.
+  <emphasis>name</emphasis> to <emphasis>value</emphasis>.
   This overrides settings in the Nix configuration file (see
   <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>).</para></listitem>
 
diff --git a/doc/manual/command-ref/opt-inst-syn.xml b/doc/manual/command-ref/opt-inst-syn.xml
index e8c3f1ec6..e324a5e6d 100644
--- a/doc/manual/command-ref/opt-inst-syn.xml
+++ b/doc/manual/command-ref/opt-inst-syn.xml
@@ -17,6 +17,6 @@
   <arg><option>--from-expression</option></arg>
   <arg><option>-E</option></arg>
     
-  <arg><option>--from-profile</option> <replaceable>path</replaceable></arg>
+  <arg><option>--from-profile</option> <emphasis>path</emphasis></arg>
 
 </nop>
diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 3a0413ceb..794d020d9 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -91,12 +91,12 @@ disallowedRequisites = [ foobar ];
     references graph of their inputs.  The attribute is a list of
     inputs in the Nix store whose references graph the builder needs
     to know.  The value of this attribute should be a list of pairs
-    <literal>[ <replaceable>name1</replaceable>
-    <replaceable>path1</replaceable> <replaceable>name2</replaceable>
-    <replaceable>path2</replaceable> <replaceable>...</replaceable>
+    <literal>[ <emphasis>name1</emphasis>
+    <emphasis>path1</emphasis> <emphasis>name2</emphasis>
+    <emphasis>path2</emphasis> <emphasis>...</emphasis>
     ]</literal>.  The references graph of each
-    <replaceable>pathN</replaceable> will be stored in a text file
-    <replaceable>nameN</replaceable> in the temporary build directory.
+    <emphasis>pathN</emphasis> will be stored in a text file
+    <emphasis>nameN</emphasis> in the temporary build directory.
     The text files have the format used by <command>nix-store
     --register-validity</command> (with the deriver fields left
     empty).  For example, when the following derivation is built:
@@ -135,7 +135,7 @@ derivation {
     Nixpkgs has the line
 
 <programlisting>
-impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
+impureEnvVars = [ "http_proxy" "https_proxy" <emphasis>...</emphasis> ];
 </programlisting>
 
     to make it use the proxy server configuration specified by the
@@ -297,11 +297,11 @@ big = "a very long string";
     <literal>bigPath</literal> will contain the absolute path to a
     temporary file containing <literal>a very long
     string</literal>. That is, for any attribute
-    <replaceable>x</replaceable> listed in
+    <emphasis>x</emphasis> listed in
     <varname>passAsFile</varname>, Nix will pass an environment
-    variable <literal><replaceable>x</replaceable>Path</literal> holding
+    variable <literal><emphasis>x</emphasis>Path</literal> holding
     the path of the file containing the value of attribute
-    <replaceable>x</replaceable>. This is useful when you need to pass
+    <emphasis>x</emphasis>. This is useful when you need to pass
     large strings to a builder, since most operating systems impose a
     limit on the size of the environment (typically, a few hundred
     kilobyte).</para></listitem>
diff --git a/doc/manual/expressions/build-script.xml b/doc/manual/expressions/build-script.xml
index 892274f50..c9af198f2 100644
--- a/doc/manual/expressions/build-script.xml
+++ b/doc/manual/expressions/build-script.xml
@@ -58,7 +58,7 @@ steps:</para>
     the <literal>PATH</literal>.  The <literal>perl</literal> environment
     variable points to the location of the Perl package (since it
     was passed in as an attribute to the derivation), so
-    <filename><replaceable>$perl</replaceable>/bin</filename> is the
+    <filename><emphasis>$perl</emphasis>/bin</filename> is the
     directory containing the Perl interpreter.</para>
 
   </listitem>
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 877fd3a1b..4ad481ad3 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -22,34 +22,34 @@ available as <function>builtins.derivation</function>.</para>
 
 
   <varlistentry xml:id='builtin-abort'>
-    <term><function>abort</function> <replaceable>s</replaceable></term>
-    <term><function>builtins.abort</function> <replaceable>s</replaceable></term>
+    <term><function>abort</function> <emphasis>s</emphasis></term>
+    <term><function>builtins.abort</function> <emphasis>s</emphasis></term>
 
     <listitem><para>Abort Nix expression evaluation, print error
-    message <replaceable>s</replaceable>.</para></listitem>
+    message <emphasis>s</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-add'>
     <term><function>builtins.add</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis>
     </term>
 
     <listitem><para>Return the sum of the numbers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-all'>
     <term><function>builtins.all</function>
-    <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>pred</emphasis> <emphasis>list</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if the function
-    <replaceable>pred</replaceable> returns <literal>true</literal>
-    for all elements of <replaceable>list</replaceable>,
+    <emphasis>pred</emphasis> returns <literal>true</literal>
+    for all elements of <emphasis>list</emphasis>,
     and <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -57,11 +57,11 @@ available as <function>builtins.derivation</function>.</para>
 
   <varlistentry xml:id='builtin-any'>
     <term><function>builtins.any</function>
-    <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>pred</emphasis> <emphasis>list</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if the function
-    <replaceable>pred</replaceable> returns <literal>true</literal>
-    for at least one element of <replaceable>list</replaceable>,
+    <emphasis>pred</emphasis> returns <literal>true</literal>
+    for at least one element of <emphasis>list</emphasis>,
     and <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -69,10 +69,10 @@ available as <function>builtins.derivation</function>.</para>
 
   <varlistentry xml:id='builtin-attrNames'>
     <term><function>builtins.attrNames</function>
-    <replaceable>set</replaceable></term>
+    <emphasis>set</emphasis></term>
 
     <listitem><para>Return the names of the attributes in the set
-    <replaceable>set</replaceable> in an alphabetically sorted list.  For instance,
+    <emphasis>set</emphasis> in an alphabetically sorted list.  For instance,
     <literal>builtins.attrNames { y = 1; x = "foo"; }</literal>
     evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem>
 
@@ -81,20 +81,20 @@ available as <function>builtins.derivation</function>.</para>
 
   <varlistentry xml:id='builtin-attrValues'>
     <term><function>builtins.attrValues</function>
-    <replaceable>set</replaceable></term>
+    <emphasis>set</emphasis></term>
 
     <listitem><para>Return the values of the attributes in the set
-    <replaceable>set</replaceable> in the order corresponding to the
+    <emphasis>set</emphasis> in the order corresponding to the
     sorted attribute names.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-baseNameOf'>
-    <term><function>baseNameOf</function> <replaceable>s</replaceable></term>
+    <term><function>baseNameOf</function> <emphasis>s</emphasis></term>
 
     <listitem><para>Return the <emphasis>base name</emphasis> of the
-    string <replaceable>s</replaceable>, that is, everything following
+    string <emphasis>s</emphasis>, that is, everything following
     the final slash in the string.  This is similar to the GNU
     <command>basename</command> command.</para></listitem>
 
@@ -103,33 +103,33 @@ available as <function>builtins.derivation</function>.</para>
 
   <varlistentry xml:id='builtin-bitAnd'>
     <term><function>builtins.bitAnd</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return the bitwise AND of the integers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-bitOr'>
     <term><function>builtins.bitOr</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return the bitwise OR of the integers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-bitXor'>
     <term><function>builtins.bitXor</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return the bitwise XOR of the integers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
@@ -154,15 +154,15 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-compareVersions'>
     <term><function>builtins.compareVersions</function>
-    <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
+    <emphasis>s1</emphasis> <emphasis>s2</emphasis></term>
 
     <listitem><para>Compare two strings representing versions and
     return <literal>-1</literal> if version
-    <replaceable>s1</replaceable> is older than version
-    <replaceable>s2</replaceable>, <literal>0</literal> if they are
+    <emphasis>s1</emphasis> is older than version
+    <emphasis>s2</emphasis>, <literal>0</literal> if they are
     the same, and <literal>1</literal> if
-    <replaceable>s1</replaceable> is newer than
-    <replaceable>s2</replaceable>.  The version comparison algorithm
+    <emphasis>s1</emphasis> is newer than
+    <emphasis>s2</emphasis>.  The version comparison algorithm
     is the same as the one used by <link
     linkend="ssec-version-comparisons"><command>nix-env
     -u</command></link>.</para></listitem>
@@ -172,7 +172,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-concatLists'>
     <term><function>builtins.concatLists</function>
-    <replaceable>lists</replaceable></term>
+    <emphasis>lists</emphasis></term>
 
     <listitem><para>Concatenate a list of lists into a single
     list.</para></listitem>
@@ -181,7 +181,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-concatStringsSep'>
     <term><function>builtins.concatStringsSep</function>
-    <replaceable>separator</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>separator</emphasis> <emphasis>list</emphasis></term>
 
     <listitem><para>Concatenate a list of strings with a separator
     between each element, e.g. <literal>concatStringsSep "/"
@@ -225,12 +225,12 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-deepSeq'>
     <term><function>builtins.deepSeq</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>This is like <literal>seq
-    <replaceable>e1</replaceable>
-    <replaceable>e2</replaceable></literal>, except that
-    <replaceable>e1</replaceable> is evaluated
+    <emphasis>e1</emphasis>
+    <emphasis>e2</emphasis></literal>, except that
+    <emphasis>e1</emphasis> is evaluated
     <emphasis>deeply</emphasis>: if it’s a list or set, its elements
     or attributes are also evaluated recursively.</para></listitem>
 
@@ -239,9 +239,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-derivation'>
     <term><function>derivation</function>
-    <replaceable>attrs</replaceable></term>
+    <emphasis>attrs</emphasis></term>
     <term><function>builtins.derivation</function>
-    <replaceable>attrs</replaceable></term>
+    <emphasis>attrs</emphasis></term>
 
     <listitem><para><function>derivation</function> is described in
     <xref linkend='ssec-derivation' />.</para></listitem>
@@ -250,11 +250,11 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
 
   <varlistentry xml:id='builtin-dirOf'>
-    <term><function>dirOf</function> <replaceable>s</replaceable></term>
-    <term><function>builtins.dirOf</function> <replaceable>s</replaceable></term>
+    <term><function>dirOf</function> <emphasis>s</emphasis></term>
+    <term><function>builtins.dirOf</function> <emphasis>s</emphasis></term>
 
     <listitem><para>Return the directory part of the string
-    <replaceable>s</replaceable>, that is, everything before the final
+    <emphasis>s</emphasis>, that is, everything before the final
     slash in the string.  This is similar to the GNU
     <command>dirname</command> command.</para></listitem>
 
@@ -263,21 +263,21 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-div'>
     <term><function>builtins.div</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return the quotient of the numbers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
   <varlistentry xml:id='builtin-elem'>
     <term><function>builtins.elem</function>
-    <replaceable>x</replaceable> <replaceable>xs</replaceable></term>
+    <emphasis>x</emphasis> <emphasis>xs</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if a value equal to
-    <replaceable>x</replaceable> occurs in the list
-    <replaceable>xs</replaceable>, and <literal>false</literal>
+    <emphasis>x</emphasis> occurs in the list
+    <emphasis>xs</emphasis>, and <literal>false</literal>
     otherwise.</para></listitem>
 
   </varlistentry>
@@ -285,10 +285,10 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-elemAt'>
     <term><function>builtins.elemAt</function>
-    <replaceable>xs</replaceable> <replaceable>n</replaceable></term>
+    <emphasis>xs</emphasis> <emphasis>n</emphasis></term>
 
-    <listitem><para>Return element <replaceable>n</replaceable> from
-    the list <replaceable>xs</replaceable>.  Elements are counted
+    <listitem><para>Return element <emphasis>n</emphasis> from
+    the list <emphasis>xs</emphasis>.  Elements are counted
     starting from 0.  A fatal error occurs if the index is out of
     bounds.</para></listitem>
 
@@ -297,7 +297,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-fetchurl'>
     <term><function>builtins.fetchurl</function>
-    <replaceable>url</replaceable></term>
+    <emphasis>url</emphasis></term>
 
     <listitem><para>Download the specified URL and return the path of
     the downloaded file. This function is not available if <link
@@ -309,9 +309,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-fetchTarball'>
     <term><function>fetchTarball</function>
-    <replaceable>url</replaceable></term>
+    <emphasis>url</emphasis></term>
     <term><function>builtins.fetchTarball</function>
-    <replaceable>url</replaceable></term>
+    <emphasis>url</emphasis></term>
 
     <listitem><para>Download the specified URL, unpack it and return
     the path of the unpacked tree. The file must be a tape archive
@@ -333,9 +333,9 @@ stdenv.mkDerivation { … }
     <para>The fetched tarball is cached for a certain amount of time
     (1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>.
     You can change the cache timeout either on the command line with
-    <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or
+    <option>--option tarball-ttl <emphasis>number of seconds</emphasis></option> or
     in the Nix configuration file with this option:
-    <literal><xref linkend="conf-tarball-ttl" /> <replaceable>number of seconds to cache</replaceable></literal>.
+    <literal><xref linkend="conf-tarball-ttl" /> <emphasis>number of seconds to cache</emphasis></literal>.
     </para>
 
     <para>Note that when obtaining the hash with <varname>nix-prefetch-url
@@ -367,12 +367,12 @@ stdenv.mkDerivation { … }
   <varlistentry xml:id='builtin-fetchGit'>
     <term>
       <function>builtins.fetchGit</function>
-      <replaceable>args</replaceable>
+      <emphasis>args</emphasis>
     </term>
 
     <listitem>
       <para>
-        Fetch a path from git. <replaceable>args</replaceable> can be
+        Fetch a path from git. <emphasis>args</emphasis> can be
         a URL, in which case the HEAD of the repo at that URL is
         fetched. Otherwise, it can be an attribute with the following
         attributes (all except <varname>url</varname> optional):
@@ -526,11 +526,11 @@ stdenv.mkDerivation { … }
 
 
   <varlistentry><term><function>builtins.filter</function>
-  <replaceable>f</replaceable> <replaceable>xs</replaceable></term>
+  <emphasis>f</emphasis> <emphasis>xs</emphasis></term>
 
     <listitem><para>Return a list consisting of the elements of
-    <replaceable>xs</replaceable> for which the function
-    <replaceable>f</replaceable> returns
+    <emphasis>xs</emphasis> for which the function
+    <emphasis>f</emphasis> returns
     <literal>true</literal>.</para></listitem>
 
   </varlistentry>
@@ -538,7 +538,7 @@ stdenv.mkDerivation { … }
 
   <varlistentry xml:id='builtin-filterSource'>
     <term><function>builtins.filterSource</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem>
 
@@ -569,10 +569,10 @@ stdenv.mkDerivation {
 
       </para>
 
-      <para>Thus, the first argument <replaceable>e1</replaceable>
+      <para>Thus, the first argument <emphasis>e1</emphasis>
       must be a predicate function that is called for each regular
       file, directory or symlink in the source tree
-      <replaceable>e2</replaceable>.  If the function returns
+      <emphasis>e2</emphasis>.  If the function returns
       <literal>true</literal>, the file is copied to the Nix store,
       otherwise it is omitted.  The function is called with two
       arguments.  The first is the full path of the file.  The second
@@ -584,7 +584,7 @@ stdenv.mkDerivation {
       the Nix store, so if the predicate returns
       <literal>true</literal> for them, the copy will fail). If you
       exclude a directory, the entire corresponding subtree of
-      <replaceable>e2</replaceable> will be excluded.</para>
+      <emphasis>e2</emphasis> will be excluded.</para>
 
     </listitem>
 
@@ -593,7 +593,7 @@ stdenv.mkDerivation {
 
   <varlistentry xml:id='builtin-foldl-prime'>
     <term><function>builtins.foldl’</function>
-    <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>op</emphasis> <emphasis>nul</emphasis> <emphasis>list</emphasis></term>
 
     <listitem><para>Reduce a list by applying a binary operator, from
     left to right, e.g. <literal>foldl’ op nul [x0 x1 x2 ...] = op (op
@@ -607,11 +607,11 @@ stdenv.mkDerivation {
 
   <varlistentry xml:id='builtin-functionArgs'>
     <term><function>builtins.functionArgs</function>
-    <replaceable>f</replaceable></term>
+    <emphasis>f</emphasis></term>
 
     <listitem><para>
     Return a set containing the names of the formal arguments expected
-    by the function <replaceable>f</replaceable>.
+    by the function <emphasis>f</emphasis>.
     The value of each attribute is a Boolean denoting whether the corresponding
     argument has a default value.  For instance,
     <literal>functionArgs ({ x, y ? 123}: ...)  =  { x = false; y = true; }</literal>.
@@ -625,7 +625,7 @@ stdenv.mkDerivation {
 
 
   <varlistentry xml:id='builtin-fromJSON'>
-    <term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term>
+    <term><function>builtins.fromJSON</function> <emphasis>e</emphasis></term>
 
     <listitem><para>Convert a JSON string to a Nix
     value. For example,
@@ -642,12 +642,12 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
 
   <varlistentry xml:id='builtin-genList'>
     <term><function>builtins.genList</function>
-    <replaceable>generator</replaceable> <replaceable>length</replaceable></term>
+    <emphasis>generator</emphasis> <emphasis>length</emphasis></term>
 
     <listitem><para>Generate list of size
-    <replaceable>length</replaceable>, with each element
-    <replaceable>i</replaceable> equal to the value returned by
-    <replaceable>generator</replaceable> <literal>i</literal>. For
+    <emphasis>length</emphasis>, with each element
+    <emphasis>i</emphasis> equal to the value returned by
+    <emphasis>generator</emphasis> <literal>i</literal>. For
     example,
 
 <programlisting>
@@ -661,13 +661,13 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-getAttr'>
     <term><function>builtins.getAttr</function>
-    <replaceable>s</replaceable> <replaceable>set</replaceable></term>
+    <emphasis>s</emphasis> <emphasis>set</emphasis></term>
 
     <listitem><para><function>getAttr</function> returns the attribute
-    named <replaceable>s</replaceable> from
-    <replaceable>set</replaceable>.  Evaluation aborts if the
+    named <emphasis>s</emphasis> from
+    <emphasis>set</emphasis>.  Evaluation aborts if the
     attribute doesn’t exist.  This is a dynamic version of the
-    <literal>.</literal> operator, since <replaceable>s</replaceable>
+    <literal>.</literal> operator, since <emphasis>s</emphasis>
     is an expression rather than an identifier.</para></listitem>
 
   </varlistentry>
@@ -675,10 +675,10 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-getEnv'>
     <term><function>builtins.getEnv</function>
-    <replaceable>s</replaceable></term>
+    <emphasis>s</emphasis></term>
 
     <listitem><para><function>getEnv</function> returns the value of
-    the environment variable <replaceable>s</replaceable>, or an empty
+    the environment variable <emphasis>s</emphasis>, or an empty
     string if the variable doesn’t exist.  This function should be
     used with care, as it can introduce all sorts of nasty environment
     dependencies in your Nix expression.</para>
@@ -694,14 +694,14 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-hasAttr'>
     <term><function>builtins.hasAttr</function>
-    <replaceable>s</replaceable> <replaceable>set</replaceable></term>
+    <emphasis>s</emphasis> <emphasis>set</emphasis></term>
 
     <listitem><para><function>hasAttr</function> returns
-    <literal>true</literal> if <replaceable>set</replaceable> has an
-    attribute named <replaceable>s</replaceable>, and
+    <literal>true</literal> if <emphasis>set</emphasis> has an
+    attribute named <emphasis>s</emphasis>, and
     <literal>false</literal> otherwise.  This is a dynamic version of
     the <literal>?</literal>  operator, since
-    <replaceable>s</replaceable> is an expression rather than an
+    <emphasis>s</emphasis> is an expression rather than an
     identifier.</para></listitem>
 
   </varlistentry>
@@ -709,11 +709,11 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-hashString'>
     <term><function>builtins.hashString</function>
-    <replaceable>type</replaceable> <replaceable>s</replaceable></term>
+    <emphasis>type</emphasis> <emphasis>s</emphasis></term>
 
     <listitem><para>Return a base-16 representation of the
-    cryptographic hash of string <replaceable>s</replaceable>.  The
-    hash algorithm specified by <replaceable>type</replaceable> must
+    cryptographic hash of string <emphasis>s</emphasis>.  The
+    hash algorithm specified by <emphasis>type</emphasis> must
     be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
     <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
 
@@ -722,11 +722,11 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-hashFile'>
     <term><function>builtins.hashFile</function>
-    <replaceable>type</replaceable> <replaceable>p</replaceable></term>
+    <emphasis>type</emphasis> <emphasis>p</emphasis></term>
 
     <listitem><para>Return a base-16 representation of the
-    cryptographic hash of the file at path <replaceable>p</replaceable>.  The
-    hash algorithm specified by <replaceable>type</replaceable> must
+    cryptographic hash of the file at path <emphasis>p</emphasis>.  The
+    hash algorithm specified by <emphasis>type</emphasis> must
     be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
     <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
 
@@ -735,7 +735,7 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-head'>
     <term><function>builtins.head</function>
-    <replaceable>list</replaceable></term>
+    <emphasis>list</emphasis></term>
 
     <listitem><para>Return the first element of a list; abort
     evaluation if the argument isn’t a list or is an empty list.  You
@@ -747,13 +747,13 @@ builtins.genList (x: x * x) 5
 
   <varlistentry xml:id='builtin-import'>
     <term><function>import</function>
-    <replaceable>path</replaceable></term>
+    <emphasis>path</emphasis></term>
     <term><function>builtins.import</function>
-    <replaceable>path</replaceable></term>
+    <emphasis>path</emphasis></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
-    file <replaceable>path</replaceable>.  If <replaceable>path
-    </replaceable> is a directory, the file <filename>default.nix
+    file <emphasis>path</emphasis>.  If <emphasis>path
+    </emphasis> is a directory, the file <filename>default.nix
     </filename> in that directory is loaded.  Evaluation aborts if the
     file doesn’t exist or contains an incorrect Nix expression.
     <function>import</function> implements Nix’s module system: you
@@ -763,7 +763,7 @@ builtins.genList (x: x * x) 5
 
     <note><para>Unlike some languages, <function>import</function> is a regular
     function in Nix. Paths using the angle bracket syntax (e.g., <function>
-    import</function> <replaceable>&lt;foo&gt;</replaceable>) are normal path
+    import</function> <emphasis>&lt;foo&gt;</emphasis>) are normal path
     values (see <xref linkend='ssec-values' />).</para></note>
 
     <para>A Nix expression loaded by <function>import</function> must
@@ -810,21 +810,21 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-intersectAttrs'>
     <term><function>builtins.intersectAttrs</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return a set consisting of the attributes in the
-    set <replaceable>e2</replaceable> that also exist in the set
-    <replaceable>e1</replaceable>.</para></listitem>
+    set <emphasis>e2</emphasis> that also exist in the set
+    <emphasis>e1</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-isAttrs'>
     <term><function>builtins.isAttrs</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a set, and
+    <emphasis>e</emphasis> evaluates to a set, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -832,20 +832,20 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-isList'>
     <term><function>builtins.isList</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a list, and
+    <emphasis>e</emphasis> evaluates to a list, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function>
-  <replaceable>e</replaceable></term>
+  <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a function, and
+    <emphasis>e</emphasis> evaluates to a function, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -853,10 +853,10 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-isString'>
     <term><function>builtins.isString</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a string, and
+    <emphasis>e</emphasis> evaluates to a string, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -864,10 +864,10 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-isInt'>
     <term><function>builtins.isInt</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to an int, and
+    <emphasis>e</emphasis> evaluates to an int, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -875,10 +875,10 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-isFloat'>
     <term><function>builtins.isFloat</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a float, and
+    <emphasis>e</emphasis> evaluates to a float, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -886,31 +886,31 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-isBool'>
     <term><function>builtins.isBool</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a bool, and
+    <emphasis>e</emphasis> evaluates to a bool, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
 
   <varlistentry><term><function>builtins.isPath</function>
-  <replaceable>e</replaceable></term>
+  <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a path, and
+    <emphasis>e</emphasis> evaluates to a path, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
 
   <varlistentry xml:id='builtin-isNull'>
     <term><function>isNull</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
     <term><function>builtins.isNull</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to <literal>null</literal>,
+    <emphasis>e</emphasis> evaluates to <literal>null</literal>,
     and <literal>false</literal> otherwise.</para>
 
     <warning><para>This function is <emphasis>deprecated</emphasis>;
@@ -923,23 +923,23 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-length'>
     <term><function>builtins.length</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return the length of the list
-    <replaceable>e</replaceable>.</para></listitem>
+    <emphasis>e</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-lessThan'>
     <term><function>builtins.lessThan</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if the number
-    <replaceable>e1</replaceable> is less than the number
-    <replaceable>e2</replaceable>, and <literal>false</literal>
+    <emphasis>e1</emphasis> is less than the number
+    <emphasis>e2</emphasis>, and <literal>false</literal>
     otherwise.  Evaluation aborts if either
-    <replaceable>e1</replaceable> or <replaceable>e2</replaceable>
+    <emphasis>e1</emphasis> or <emphasis>e2</emphasis>
     does not evaluate to a number.</para></listitem>
 
   </varlistentry>
@@ -947,7 +947,7 @@ x: x + 456</programlisting>
 
   <varlistentry xml:id='builtin-listToAttrs'>
     <term><function>builtins.listToAttrs</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Construct a set from a list specifying the names
     and values of each attribute.  Each element of the list should be
@@ -975,12 +975,12 @@ builtins.listToAttrs
 
   <varlistentry xml:id='builtin-map'>
     <term><function>map</function>
-    <replaceable>f</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>f</emphasis> <emphasis>list</emphasis></term>
     <term><function>builtins.map</function>
-    <replaceable>f</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>f</emphasis> <emphasis>list</emphasis></term>
 
-    <listitem><para>Apply the function <replaceable>f</replaceable> to
-    each element in the list <replaceable>list</replaceable>.  For
+    <listitem><para>Apply the function <emphasis>f</emphasis> to
+    each element in the list <emphasis>list</emphasis>.  For
     example,
 
 <programlisting>
@@ -994,12 +994,12 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting>
 
   <varlistentry xml:id='builtin-match'>
     <term><function>builtins.match</function>
-    <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+    <emphasis>regex</emphasis> <emphasis>str</emphasis></term>
 
     <listitem><para>Returns a list if the <link
     xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-    POSIX regular expression</link> <replaceable>regex</replaceable>
-    matches <replaceable>str</replaceable> precisely, otherwise returns
+    POSIX regular expression</link> <emphasis>regex</emphasis>
+    matches <emphasis>str</emphasis> precisely, otherwise returns
     <literal>null</literal>.  Each item in the list is a regex group.
 
 <programlisting>
@@ -1031,20 +1031,20 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   <varlistentry xml:id='builtin-mul'>
     <term><function>builtins.mul</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return the product of the numbers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-parseDrvName'>
     <term><function>builtins.parseDrvName</function>
-    <replaceable>s</replaceable></term>
+    <emphasis>s</emphasis></term>
 
-    <listitem><para>Split the string <replaceable>s</replaceable> into
+    <listitem><para>Split the string <emphasis>s</emphasis> into
     a package name and version.  The package name is everything up to
     but not including the first dash followed by a digit, and the
     version is everything following that dash.  The result is returned
@@ -1058,13 +1058,13 @@ Evaluates to <literal>[ "foo" ]</literal>.
   <varlistentry xml:id='builtin-path'>
     <term>
       <function>builtins.path</function>
-      <replaceable>args</replaceable>
+      <emphasis>args</emphasis>
     </term>
 
     <listitem>
       <para>
         An enrichment of the built-in path type, based on the attributes
-        present in <replaceable>args</replaceable>. All are optional
+        present in <emphasis>args</emphasis>. All are optional
         except <varname>path</varname>:
       </para>
 
@@ -1127,20 +1127,20 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   <varlistentry xml:id='builtin-pathExists'>
     <term><function>builtins.pathExists</function>
-    <replaceable>path</replaceable></term>
+    <emphasis>path</emphasis></term>
 
     <listitem><para>Return <literal>true</literal> if the path
-    <replaceable>path</replaceable> exists at evaluation time, and
+    <emphasis>path</emphasis> exists at evaluation time, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
 
   <varlistentry xml:id='builtin-placeholder'>
     <term><function>builtins.placeholder</function>
-    <replaceable>output</replaceable></term>
+    <emphasis>output</emphasis></term>
 
     <listitem><para>Return a placeholder string for the specified
-    <replaceable>output</replaceable> that will be substituted by the
+    <emphasis>output</emphasis> that will be substituted by the
     corresponding output path at build time. Typical outputs would be
     <literal>"out"</literal>, <literal>"bin"</literal> or
     <literal>"dev"</literal>.</para></listitem>
@@ -1148,10 +1148,10 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   <varlistentry xml:id='builtin-readDir'>
     <term><function>builtins.readDir</function>
-    <replaceable>path</replaceable></term>
+    <emphasis>path</emphasis></term>
 
     <listitem><para>Return the contents of the directory
-    <replaceable>path</replaceable> as a set mapping directory entries
+    <emphasis>path</emphasis> as a set mapping directory entries
     to the corresponding file type. For instance, if directory
     <filename>A</filename> contains a regular file
     <filename>B</filename> and another directory
@@ -1171,24 +1171,24 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   <varlistentry xml:id='builtin-readFile'>
     <term><function>builtins.readFile</function>
-    <replaceable>path</replaceable></term>
+    <emphasis>path</emphasis></term>
 
     <listitem><para>Return the contents of the file
-    <replaceable>path</replaceable> as a string.</para></listitem>
+    <emphasis>path</emphasis> as a string.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-removeAttrs'>
     <term><function>removeAttrs</function>
-    <replaceable>set</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>set</emphasis> <emphasis>list</emphasis></term>
     <term><function>builtins.removeAttrs</function>
-    <replaceable>set</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>set</emphasis> <emphasis>list</emphasis></term>
 
     <listitem><para>Remove the attributes listed in
-    <replaceable>list</replaceable> from
-    <replaceable>set</replaceable>.  The attributes don’t have to
-    exist in <replaceable>set</replaceable>. For instance,
+    <emphasis>list</emphasis> from
+    <emphasis>set</emphasis>.  The attributes don’t have to
+    exist in <emphasis>set</emphasis>. For instance,
 
 <programlisting>
 removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting>
@@ -1200,12 +1200,12 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting>
 
   <varlistentry xml:id='builtin-replaceStrings'>
     <term><function>builtins.replaceStrings</function>
-    <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term>
+    <emphasis>from</emphasis> <emphasis>to</emphasis> <emphasis>s</emphasis></term>
 
-    <listitem><para>Given string <replaceable>s</replaceable>, replace
-    every occurrence of the strings in <replaceable>from</replaceable>
+    <listitem><para>Given string <emphasis>s</emphasis>, replace
+    every occurrence of the strings in <emphasis>from</emphasis>
     with the corresponding string in
-    <replaceable>to</replaceable>. For example,
+    <emphasis>to</emphasis>. For example,
 
 <programlisting>
 builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
@@ -1218,23 +1218,23 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
 
   <varlistentry xml:id='builtin-seq'>
     <term><function>builtins.seq</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
-    <listitem><para>Evaluate <replaceable>e1</replaceable>, then
-    evaluate and return <replaceable>e2</replaceable>. This ensures
+    <listitem><para>Evaluate <emphasis>e1</emphasis>, then
+    evaluate and return <emphasis>e2</emphasis>. This ensures
     that a computation is strict in the value of
-    <replaceable>e1</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-sort'>
     <term><function>builtins.sort</function>
-    <replaceable>comparator</replaceable> <replaceable>list</replaceable></term>
+    <emphasis>comparator</emphasis> <emphasis>list</emphasis></term>
 
-    <listitem><para>Return <replaceable>list</replaceable> in sorted
+    <listitem><para>Return <emphasis>list</emphasis> in sorted
     order. It repeatedly calls the function
-    <replaceable>comparator</replaceable> with two elements. The
+    <emphasis>comparator</emphasis> with two elements. The
     comparator should return <literal>true</literal> if the first
     element is less than the second, and <literal>false</literal>
     otherwise. For example,
@@ -1254,13 +1254,13 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
 
   <varlistentry xml:id='builtin-split'>
     <term><function>builtins.split</function>
-    <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+    <emphasis>regex</emphasis> <emphasis>str</emphasis></term>
 
     <listitem><para>Returns a list composed of non matched strings interleaved
     with the lists of the <link
     xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-    POSIX regular expression</link> <replaceable>regex</replaceable> matches
-    of <replaceable>str</replaceable>. Each item in the lists of matched
+    POSIX regular expression</link> <emphasis>regex</emphasis> matches
+    of <emphasis>str</emphasis>. Each item in the lists of matched
     sequences is a regex group.
 
 <programlisting>
@@ -1293,7 +1293,7 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
 
   <varlistentry xml:id='builtin-splitVersion'>
     <term><function>builtins.splitVersion</function>
-    <replaceable>s</replaceable></term>
+    <emphasis>s</emphasis></term>
 
     <listitem><para>Split a string representing a version into its
     components, by the same version splitting logic underlying the
@@ -1305,10 +1305,10 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
 
   <varlistentry xml:id='builtin-stringLength'>
     <term><function>builtins.stringLength</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return the length of the string
-    <replaceable>e</replaceable>.  If <replaceable>e</replaceable> is
+    <emphasis>e</emphasis>.  If <emphasis>e</emphasis> is
     not a string, evaluation is aborted.</para></listitem>
 
   </varlistentry>
@@ -1316,29 +1316,29 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
 
   <varlistentry xml:id='builtin-sub'>
     <term><function>builtins.sub</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
     <listitem><para>Return the difference between the numbers
-    <replaceable>e1</replaceable> and
-    <replaceable>e2</replaceable>.</para></listitem>
+    <emphasis>e1</emphasis> and
+    <emphasis>e2</emphasis>.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id='builtin-substring'>
     <term><function>builtins.substring</function>
-    <replaceable>start</replaceable> <replaceable>len</replaceable>
-    <replaceable>s</replaceable></term>
+    <emphasis>start</emphasis> <emphasis>len</emphasis>
+    <emphasis>s</emphasis></term>
 
     <listitem><para>Return the substring of
-    <replaceable>s</replaceable> from character position
-    <replaceable>start</replaceable> (zero-based) up to but not
-    including <replaceable>start + len</replaceable>.  If
-    <replaceable>start</replaceable> is greater than the length of the
-    string, an empty string is returned, and if <replaceable>start +
-    len</replaceable> lies beyond the end of the string, only the
+    <emphasis>s</emphasis> from character position
+    <emphasis>start</emphasis> (zero-based) up to but not
+    including <emphasis>start + len</emphasis>.  If
+    <emphasis>start</emphasis> is greater than the length of the
+    string, an empty string is returned, and if <emphasis>start +
+    len</emphasis> lies beyond the end of the string, only the
     substring up to the end of the string is returned.
-    <replaceable>start</replaceable> must be
+    <emphasis>start</emphasis> must be
     non-negative. For example,
 
 <programlisting>
@@ -1353,7 +1353,7 @@ builtins.substring 0 3 "nixos"
 
   <varlistentry xml:id='builtin-tail'>
     <term><function>builtins.tail</function>
-    <replaceable>list</replaceable></term>
+    <emphasis>list</emphasis></term>
 
     <listitem><para>Return the second to last elements of a list;
     abort evaluation if the argument isn’t a list or is an empty
@@ -1364,12 +1364,12 @@ builtins.substring 0 3 "nixos"
 
   <varlistentry xml:id='builtin-throw'>
     <term><function>throw</function>
-    <replaceable>s</replaceable></term>
+    <emphasis>s</emphasis></term>
     <term><function>builtins.throw</function>
-    <replaceable>s</replaceable></term>
+    <emphasis>s</emphasis></term>
 
     <listitem><para>Throw an error message
-    <replaceable>s</replaceable>.  This usually aborts Nix expression
+    <emphasis>s</emphasis>.  This usually aborts Nix expression
     evaluation, but in <command>nix-env -qa</command> and other
     commands that try to evaluate a set of derivations to get
     information about those derivations, a derivation that throws an
@@ -1381,12 +1381,12 @@ builtins.substring 0 3 "nixos"
 
   <varlistentry xml:id='builtin-toFile'>
     <term><function>builtins.toFile</function>
-    <replaceable>name</replaceable>
-    <replaceable>s</replaceable></term>
+    <emphasis>name</emphasis>
+    <emphasis>s</emphasis></term>
 
-    <listitem><para>Store the string <replaceable>s</replaceable> in a
+    <listitem><para>Store the string <emphasis>s</emphasis> in a
     file in the Nix store and return its path.  The file has suffix
-    <replaceable>name</replaceable>.  This file can be used as an
+    <emphasis>name</emphasis>.  This file can be used as an
     input to derivations.  One application is to write builders
     “inline”.  For instance, the following Nix expression combines
     <xref linkend='ex-hello-nix' /> and <xref
@@ -1425,11 +1425,11 @@ stdenv.mkDerivation {
   builder = let
     configFile = builtins.toFile "foo.conf" "
       # This is some dummy configuration file.
-      <replaceable>...</replaceable>
+      <emphasis>...</emphasis>
     ";
   in builtins.toFile "builder.sh" "
     source $stdenv/setup
-    <replaceable>...</replaceable>
+    <emphasis>...</emphasis>
     cp ${configFile} $out/etc/foo.conf
   ";</programlisting>
 
@@ -1459,10 +1459,10 @@ in foo</programlisting>
 
 
   <varlistentry xml:id='builtin-toJSON'>
-    <term><function>builtins.toJSON</function> <replaceable>e</replaceable></term>
+    <term><function>builtins.toJSON</function> <emphasis>e</emphasis></term>
 
     <listitem><para>Return a string containing a JSON representation
-    of <replaceable>e</replaceable>.  Strings, integers, floats, booleans,
+    of <emphasis>e</emphasis>.  Strings, integers, floats, booleans,
     nulls and lists are mapped to their JSON equivalents.  Sets
     (except derivations) are represented as objects.  Derivations are
     translated to a JSON string containing the derivation’s output
@@ -1473,7 +1473,7 @@ in foo</programlisting>
 
 
   <varlistentry xml:id='builtin-toPath'>
-    <term><function>builtins.toPath</function> <replaceable>s</replaceable></term>
+    <term><function>builtins.toPath</function> <emphasis>s</emphasis></term>
 
     <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal>
     to convert a string into an absolute path. For relative paths,
@@ -1484,12 +1484,12 @@ in foo</programlisting>
 
 
   <varlistentry xml:id='builtin-toString'>
-    <term><function>toString</function> <replaceable>e</replaceable></term>
-    <term><function>builtins.toString</function> <replaceable>e</replaceable></term>
+    <term><function>toString</function> <emphasis>e</emphasis></term>
+    <term><function>builtins.toString</function> <emphasis>e</emphasis></term>
 
     <listitem><para>Convert the expression
-    <replaceable>e</replaceable> to a string.
-    <replaceable>e</replaceable> can be:</para>
+    <emphasis>e</emphasis> to a string.
+    <emphasis>e</emphasis> can be:</para>
     <itemizedlist>
       <listitem><para>A string (in which case the string is returned unmodified).</para></listitem>
       <listitem><para>A path (e.g., <literal>toString /foo/bar</literal> yields <literal>"/foo/bar"</literal>.</para></listitem>
@@ -1505,10 +1505,10 @@ in foo</programlisting>
 
 
   <varlistentry xml:id='builtin-toXML'>
-    <term><function>builtins.toXML</function> <replaceable>e</replaceable></term>
+    <term><function>builtins.toXML</function> <emphasis>e</emphasis></term>
 
     <listitem><para>Return a string containing an XML representation
-    of <replaceable>e</replaceable>.  The main application for
+    of <emphasis>e</emphasis>.  The main application for
     <function>toXML</function> is to communicate information with the
     builder in a more structured format than plain environment
     variables.</para>
@@ -1612,26 +1612,26 @@ stdenv.mkDerivation (rec {
 
   <varlistentry xml:id='builtin-trace'>
     <term><function>builtins.trace</function>
-    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
 
-    <listitem><para>Evaluate <replaceable>e1</replaceable> and print its
+    <listitem><para>Evaluate <emphasis>e1</emphasis> and print its
     abstract syntax representation on standard error.  Then return
-    <replaceable>e2</replaceable>.  This function is useful for
+    <emphasis>e2</emphasis>.  This function is useful for
     debugging.</para></listitem>
 
   </varlistentry>
 
   <varlistentry xml:id='builtin-tryEval'>
     <term><function>builtins.tryEval</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
-    <listitem><para>Try to shallowly evaluate <replaceable>e</replaceable>.
+    <listitem><para>Try to shallowly evaluate <emphasis>e</emphasis>.
     Return a set containing the attributes <literal>success</literal>
-    (<literal>true</literal> if <replaceable>e</replaceable> evaluated
+    (<literal>true</literal> if <emphasis>e</emphasis> evaluated
     successfully, <literal>false</literal> if an error was thrown) and
-    <literal>value</literal>, equalling <replaceable>e</replaceable>
+    <literal>value</literal>, equalling <emphasis>e</emphasis>
     if successful and <literal>false</literal> otherwise. Note that this
-    doesn't evaluate <replaceable>e</replaceable> deeply, so
+    doesn't evaluate <emphasis>e</emphasis> deeply, so
     <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success
     </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq
     </literal> one can get the expected result: <literal>let e = { x = throw "";
@@ -1644,10 +1644,10 @@ stdenv.mkDerivation (rec {
 
   <varlistentry xml:id='builtin-typeOf'>
     <term><function>builtins.typeOf</function>
-    <replaceable>e</replaceable></term>
+    <emphasis>e</emphasis></term>
 
     <listitem><para>Return a string representing the type of the value
-    <replaceable>e</replaceable>, namely <literal>"int"</literal>,
+    <emphasis>e</emphasis>, namely <literal>"int"</literal>,
     <literal>"bool"</literal>, <literal>"string"</literal>,
     <literal>"path"</literal>, <literal>"null"</literal>,
     <literal>"set"</literal>, <literal>"list"</literal>,
diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml
index 562a9e9f4..853e40b58 100644
--- a/doc/manual/expressions/expression-syntax.xml
+++ b/doc/manual/expressions/expression-syntax.xml
@@ -50,7 +50,7 @@ the single Nix expression in that directory
     <para>Nix functions generally have the form <literal>{ x, y, ...,
     z }: e</literal> where <varname>x</varname>, <varname>y</varname>,
     etc. are the names of the expected arguments, and where
-    <replaceable>e</replaceable> is the body of the function.  So
+    <emphasis>e</emphasis> is the body of the function.  So
     here, the entire remainder of the file is the body of the
     function; when given the required arguments, the body should
     describe how to build an instance of the Hello package.</para>
@@ -69,10 +69,10 @@ the single Nix expression in that directory
     <emphasis>attributes</emphasis>.  A set is just a list of
     key/value pairs where each key is a string and each value is an
     arbitrary Nix expression.  They take the general form <literal>{
-    <replaceable>name1</replaceable> =
-    <replaceable>expr1</replaceable>; <replaceable>...</replaceable>
-    <replaceable>nameN</replaceable> =
-    <replaceable>exprN</replaceable>; }</literal>.</para>
+    <emphasis>name1</emphasis> =
+    <emphasis>expr1</emphasis>; <emphasis>...</emphasis>
+    <emphasis>nameN</emphasis> =
+    <emphasis>exprN</emphasis>; }</literal>.</para>
 
   </listitem>
 
diff --git a/doc/manual/expressions/generic-builder.xml b/doc/manual/expressions/generic-builder.xml
index 71b342f99..f6e67813d 100644
--- a/doc/manual/expressions/generic-builder.xml
+++ b/doc/manual/expressions/generic-builder.xml
@@ -44,7 +44,7 @@ genericBuild ③</programlisting>
     subdirectory, it's added to GCC's header search path; and so
     on.<footnote><para>How does it work? <filename>setup</filename>
     tries to source the file
-    <filename><replaceable>pkg</replaceable>/nix-support/setup-hook</filename>
+    <filename><emphasis>pkg</emphasis>/nix-support/setup-hook</filename>
     of all dependencies.  These “setup hooks” can then set up whatever
     environment variables they want; for instance, the setup hook for
     Perl sets the <literal>PERL5LIB</literal> environment variable to
diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 82d3afed1..86c25d723 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -137,7 +137,7 @@ while defining a set.</para>
 <para>Functions have the following form:
 
 <programlisting>
-<replaceable>pattern</replaceable>: <replaceable>body</replaceable></programlisting>
+<emphasis>pattern</emphasis>: <emphasis>body</emphasis></programlisting>
 
 The pattern specifies what the argument of the function must look
 like, and binds variables in the body to (parts of) the
@@ -190,9 +190,9 @@ map (concat "foo") [ "bar" "bla" "abc" ]</programlisting>
   <para>It is possible to provide <emphasis>default values</emphasis>
   for attributes, in which case they are allowed to be missing.  A
   default value is specified by writing
-  <literal><replaceable>name</replaceable> ?
-  <replaceable>e</replaceable></literal>, where
-  <replaceable>e</replaceable> is an arbitrary expression.  For example,
+  <literal><emphasis>name</emphasis> ?
+  <emphasis>e</emphasis></literal>, where
+  <emphasis>e</emphasis> is an arbitrary expression.  For example,
 
 <programlisting>
 { x, y ? "foo", z ? "bar" }: z + y + x</programlisting>
@@ -256,9 +256,9 @@ in concat { x = "foo"; y = "bar"; }</programlisting>
 <para>Conditionals look like this:
 
 <programlisting>
-if <replaceable>e1</replaceable> then <replaceable>e2</replaceable> else <replaceable>e3</replaceable></programlisting>
+if <emphasis>e1</emphasis> then <emphasis>e2</emphasis> else <emphasis>e3</emphasis></programlisting>
 
-where <replaceable>e1</replaceable> is an expression that should
+where <emphasis>e1</emphasis> is an expression that should
 evaluate to a Boolean value (<literal>true</literal> or
 <literal>false</literal>).</para>
 
@@ -271,11 +271,11 @@ evaluate to a Boolean value (<literal>true</literal> or
 on or between features and dependencies hold.  They look like this:
 
 <programlisting>
-assert <replaceable>e1</replaceable>; <replaceable>e2</replaceable></programlisting>
+assert <emphasis>e1</emphasis>; <emphasis>e2</emphasis></programlisting>
 
-where <replaceable>e1</replaceable> is an expression that should
+where <emphasis>e1</emphasis> is an expression that should
 evaluate to a Boolean value.  If it evaluates to
-<literal>true</literal>, <replaceable>e2</replaceable> is returned;
+<literal>true</literal>, <emphasis>e2</emphasis> is returned;
 otherwise expression evaluation is aborted and a backtrace is printed.</para>
 
 <para>Here is a Nix expression for the Subversion package that shows
@@ -358,10 +358,10 @@ stdenv.mkDerivation {
 <para>A <emphasis>with-expression</emphasis>,
 
 <programlisting>
-with <replaceable>e1</replaceable>; <replaceable>e2</replaceable></programlisting>
+with <emphasis>e1</emphasis>; <emphasis>e2</emphasis></programlisting>
 
-introduces the set <replaceable>e1</replaceable> into the lexical
-scope of the expression <replaceable>e2</replaceable>.  For instance,
+introduces the set <emphasis>e1</emphasis> into the lexical
+scope of the expression <emphasis>e2</emphasis>.  For instance,
 
 <programlisting>
 let as = { x = "foo"; y = "bar"; };
diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml
index 4f11bf529..7f69bfcef 100644
--- a/doc/manual/expressions/language-operators.xml
+++ b/doc/manual/expressions/language-operators.xml
@@ -25,48 +25,48 @@ weakest binding).</para>
     <tbody>
       <row>
         <entry>Select</entry>
-        <entry><replaceable>e</replaceable> <literal>.</literal>
-        <replaceable>attrpath</replaceable>
-        [ <literal>or</literal> <replaceable>def</replaceable> ]
+        <entry><emphasis>e</emphasis> <literal>.</literal>
+        <emphasis>attrpath</emphasis>
+        [ <literal>or</literal> <emphasis>def</emphasis> ]
         </entry>
         <entry>none</entry>
         <entry>Select attribute denoted by the attribute path
-        <replaceable>attrpath</replaceable> from set
-        <replaceable>e</replaceable>.  (An attribute path is a
+        <emphasis>attrpath</emphasis> from set
+        <emphasis>e</emphasis>.  (An attribute path is a
         dot-separated list of attribute names.)  If the attribute
-        doesn’t exist, return <replaceable>def</replaceable> if
+        doesn’t exist, return <emphasis>def</emphasis> if
         provided, otherwise abort evaluation.</entry>
         <entry>1</entry>
       </row>
       <row>
         <entry>Application</entry>
-        <entry><replaceable>e1</replaceable> <replaceable>e2</replaceable></entry>
+        <entry><emphasis>e1</emphasis> <emphasis>e2</emphasis></entry>
         <entry>left</entry>
-        <entry>Call function <replaceable>e1</replaceable> with
-        argument <replaceable>e2</replaceable>.</entry>
+        <entry>Call function <emphasis>e1</emphasis> with
+        argument <emphasis>e2</emphasis>.</entry>
         <entry>2</entry>
       </row>
       <row>
         <entry>Arithmetic Negation</entry>
-        <entry><literal>-</literal> <replaceable>e</replaceable></entry>
+        <entry><literal>-</literal> <emphasis>e</emphasis></entry>
         <entry>none</entry>
         <entry>Arithmetic negation.</entry>
         <entry>3</entry>
       </row>
       <row>
         <entry>Has Attribute</entry>
-        <entry><replaceable>e</replaceable> <literal>?</literal>
-        <replaceable>attrpath</replaceable></entry>
+        <entry><emphasis>e</emphasis> <literal>?</literal>
+        <emphasis>attrpath</emphasis></entry>
         <entry>none</entry>
-        <entry>Test whether set <replaceable>e</replaceable> contains
-        the attribute denoted by <replaceable>attrpath</replaceable>;
+        <entry>Test whether set <emphasis>e</emphasis> contains
+        the attribute denoted by <emphasis>attrpath</emphasis>;
         return <literal>true</literal> or
         <literal>false</literal>.</entry>
         <entry>4</entry>
       </row>
       <row>
         <entry>List Concatenation</entry>
-        <entry><replaceable>e1</replaceable> <literal>++</literal> <replaceable>e2</replaceable></entry>
+        <entry><emphasis>e1</emphasis> <literal>++</literal> <emphasis>e2</emphasis></entry>
         <entry>right</entry>
         <entry>List concatenation.</entry>
         <entry>5</entry>
@@ -74,7 +74,7 @@ weakest binding).</para>
       <row>
         <entry>Multiplication</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>,
+          <emphasis>e1</emphasis> <literal>*</literal> <emphasis>e2</emphasis>,
         </entry>
         <entry>left</entry>
         <entry>Arithmetic multiplication.</entry>
@@ -83,7 +83,7 @@ weakest binding).</para>
       <row>
         <entry>Division</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>/</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>left</entry>
         <entry>Arithmetic division.</entry>
@@ -92,7 +92,7 @@ weakest binding).</para>
       <row>
         <entry>Addition</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>+</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>left</entry>
         <entry>Arithmetic addition.</entry>
@@ -101,7 +101,7 @@ weakest binding).</para>
       <row>
         <entry>Subtraction</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>-</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>left</entry>
         <entry>Arithmetic subtraction.</entry>
@@ -110,7 +110,7 @@ weakest binding).</para>
       <row>
         <entry>String Concatenation</entry>
         <entry>
-          <replaceable>string1</replaceable> <literal>+</literal> <replaceable>string2</replaceable>
+          <emphasis>string1</emphasis> <literal>+</literal> <emphasis>string2</emphasis>
         </entry>
         <entry>left</entry>
         <entry>String concatenation.</entry>
@@ -118,19 +118,19 @@ weakest binding).</para>
       </row>
       <row>
         <entry>Not</entry>
-        <entry><literal>!</literal> <replaceable>e</replaceable></entry>
+        <entry><literal>!</literal> <emphasis>e</emphasis></entry>
         <entry>none</entry>
         <entry>Boolean negation.</entry>
         <entry>8</entry>
       </row>
       <row>
         <entry>Update</entry>
-        <entry><replaceable>e1</replaceable> <literal>//</literal>
-        <replaceable>e2</replaceable></entry>
+        <entry><emphasis>e1</emphasis> <literal>//</literal>
+        <emphasis>e2</emphasis></entry>
         <entry>right</entry>
         <entry>Return a set consisting of the attributes in
-        <replaceable>e1</replaceable> and
-        <replaceable>e2</replaceable> (with the latter taking
+        <emphasis>e1</emphasis> and
+        <emphasis>e2</emphasis> (with the latter taking
         precedence over the former in case of equally named
         attributes).</entry>
         <entry>9</entry>
@@ -138,7 +138,7 @@ weakest binding).</para>
       <row>
         <entry>Less Than</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>&lt;</literal> <replaceable>e2</replaceable>,
+          <emphasis>e1</emphasis> <literal>&lt;</literal> <emphasis>e2</emphasis>,
         </entry>
         <entry>none</entry>
         <entry>Arithmetic comparison.</entry>
@@ -147,7 +147,7 @@ weakest binding).</para>
       <row>
         <entry>Less Than or Equal To</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>&lt;=</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>&lt;=</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>none</entry>
         <entry>Arithmetic comparison.</entry>
@@ -156,7 +156,7 @@ weakest binding).</para>
       <row>
         <entry>Greater Than</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>&gt;</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>&gt;</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>none</entry>
         <entry>Arithmetic comparison.</entry>
@@ -165,7 +165,7 @@ weakest binding).</para>
       <row>
         <entry>Greater Than or Equal To</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>&gt;=</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>&gt;=</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>none</entry>
         <entry>Arithmetic comparison.</entry>
@@ -174,7 +174,7 @@ weakest binding).</para>
       <row>
         <entry>Equality</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>==</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>none</entry>
         <entry>Equality.</entry>
@@ -183,7 +183,7 @@ weakest binding).</para>
       <row>
         <entry>Inequality</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable>
+          <emphasis>e1</emphasis> <literal>!=</literal> <emphasis>e2</emphasis>
         </entry>
         <entry>none</entry>
         <entry>Inequality.</entry>
@@ -191,28 +191,28 @@ weakest binding).</para>
       </row>
       <row>
         <entry>Logical AND</entry>
-        <entry><replaceable>e1</replaceable> <literal>&amp;&amp;</literal>
-        <replaceable>e2</replaceable></entry>
+        <entry><emphasis>e1</emphasis> <literal>&amp;&amp;</literal>
+        <emphasis>e2</emphasis></entry>
         <entry>left</entry>
         <entry>Logical AND.</entry>
         <entry>12</entry>
       </row>
       <row>
         <entry>Logical OR</entry>
-        <entry><replaceable>e1</replaceable> <literal>||</literal>
-        <replaceable>e2</replaceable></entry>
+        <entry><emphasis>e1</emphasis> <literal>||</literal>
+        <emphasis>e2</emphasis></entry>
         <entry>left</entry>
         <entry>Logical OR.</entry>
         <entry>13</entry>
       </row>
       <row>
         <entry>Logical Implication</entry>
-        <entry><replaceable>e1</replaceable> <literal>-></literal>
-        <replaceable>e2</replaceable></entry>
+        <entry><emphasis>e1</emphasis> <literal>-></literal>
+        <emphasis>e2</emphasis></entry>
         <entry>none</entry>
         <entry>Logical implication (equivalent to
-        <literal>!<replaceable>e1</replaceable> ||
-        <replaceable>e2</replaceable></literal>).</entry>
+        <literal>!<emphasis>e1</emphasis> ||
+        <emphasis>e2</emphasis></literal>).</entry>
         <entry>14</entry>
       </row>
     </tbody>
diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml
index 6c0fcbacb..cebafb1f5 100644
--- a/doc/manual/expressions/language-values.xml
+++ b/doc/manual/expressions/language-values.xml
@@ -30,7 +30,7 @@
 
     <para>You can include the result of an expression into a string by
     enclosing it in
-    <literal>${<replaceable>...</replaceable>}</literal>, a feature
+    <literal>${<emphasis>...</emphasis>}</literal>, a feature
     known as <emphasis>antiquotation</emphasis>.  The enclosed
     expression must evaluate to something that can be coerced into a
     string (meaning that it must be a string, a path, or a
@@ -93,7 +93,7 @@ configureFlags = "
     text on the initial line.</para>
 
     <para>Antiquotation
-    (<literal>${<replaceable>expr</replaceable>}</literal>) is
+    (<literal>${<emphasis>expr</emphasis>}</literal>) is
     supported in indented strings.</para>
 
     <para>Since <literal>${</literal> and <literal>''</literal> have
@@ -119,7 +119,7 @@ configureFlags = "
 
 <programlisting>
 stdenv.mkDerivation {
-  <replaceable>...</replaceable>
+  <emphasis>...</emphasis>
   postInstall =
     ''
       mkdir $out/bin $out/etc
@@ -127,7 +127,7 @@ stdenv.mkDerivation {
       echo "Hello World" > $out/etc/foo.conf
       ${if enableBar then "cp bar $out/bin" else ""}
     '';
-  <replaceable>...</replaceable>
+  <emphasis>...</emphasis>
 }
 </programlisting>
 
diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml
index 33a802e83..f82223df9 100644
--- a/doc/manual/expressions/simple-building-testing.xml
+++ b/doc/manual/expressions/simple-building-testing.xml
@@ -19,7 +19,7 @@ building path `/nix/store/632d2b22514d...-hello-2.1.1'
 hello-2.1.1/
 hello-2.1.1/intl/
 hello-2.1.1/intl/ChangeLog
-<replaceable>...</replaceable>
+<emphasis>...</emphasis>
 
 $ ls -l result
 lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1
diff --git a/doc/manual/installation/building-source.xml b/doc/manual/installation/building-source.xml
index 772cda9cc..469aaebe9 100644
--- a/doc/manual/installation/building-source.xml
+++ b/doc/manual/installation/building-source.xml
@@ -10,7 +10,7 @@
 following commands:
 
 <screen>
-$ ./configure <replaceable>options...</replaceable>
+$ ./configure <emphasis>options...</emphasis>
 $ make
 $ make install</screen>
 
@@ -26,16 +26,16 @@ $ ./bootstrap.sh</screen>
 </para>
 
 <para>The installation path can be specified by passing the
-<option>--prefix=<replaceable>prefix</replaceable></option> to
+<option>--prefix=<emphasis>prefix</emphasis></option> to
 <command>configure</command>.  The default installation directory is
 <filename>/usr/local</filename>.  You can change this to any location
 you like.  You must have write permission to the
-<replaceable>prefix</replaceable> path.</para>
+<emphasis>prefix</emphasis> path.</para>
 
 <para>Nix keeps its <emphasis>store</emphasis> (the place where
 packages are stored) in <filename>/nix/store</filename> by default.
 This can be changed using
-<option>--with-store-dir=<replaceable>path</replaceable></option>.</para>
+<option>--with-store-dir=<emphasis>path</emphasis></option>.</para>
 
 <warning><para>It is best <emphasis>not</emphasis> to change the Nix
 store from its default, since doing so makes it impossible to use
@@ -44,6 +44,6 @@ packages will need to be built from source.</para></warning>
 
 <para>Nix keeps state (such as its database and log files) in
 <filename>/nix/var</filename> by default.  This can be changed using
-<option>--localstatedir=<replaceable>path</replaceable></option>.</para>
+<option>--localstatedir=<emphasis>path</emphasis></option>.</para>
 
 </section>
diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
index 436f15f31..cbce8559a 100644
--- a/doc/manual/installation/env-variables.xml
+++ b/doc/manual/installation/env-variables.xml
@@ -8,18 +8,18 @@
 
 <para>To use Nix, some environment variables should be set.  In
 particular, <literal>PATH</literal> should contain the directories
-<filename><replaceable>prefix</replaceable>/bin</filename> and
+<filename><emphasis>prefix</emphasis>/bin</filename> and
 <filename>~/.nix-profile/bin</filename>.  The first directory contains
 the Nix tools themselves, while <filename>~/.nix-profile</filename> is
 a symbolic link to the current <emphasis>user environment</emphasis>
 (an automatically generated package consisting of symlinks to
 installed packages).  The simplest way to set the required environment
 variables is to include the file
-<filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
+<filename><emphasis>prefix</emphasis>/etc/profile.d/nix.sh</filename>
 in your <filename>~/.profile</filename> (or similar), like this:</para>
 
 <screen>
-source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>
+source <emphasis>prefix</emphasis>/etc/profile.d/nix.sh</screen>
 
 <section xml:id="sec-nix-ssl-cert-file">
 
diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index ad47ce8b9..439198e6c 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -421,7 +421,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<emphasis>version</emphasis>/install</literal>.
   </para>
 
   <para>
diff --git a/doc/manual/installation/single-user.xml b/doc/manual/installation/single-user.xml
index 09cdaa5d4..e9a761af3 100644
--- a/doc/manual/installation/single-user.xml
+++ b/doc/manual/installation/single-user.xml
@@ -7,9 +7,9 @@
 <title>Single-User Mode</title>
 
 <para>In single-user mode, all Nix operations that access the database
-in <filename><replaceable>prefix</replaceable>/var/nix/db</filename>
+in <filename><emphasis>prefix</emphasis>/var/nix/db</filename>
 or modify the Nix store in
-<filename><replaceable>prefix</replaceable>/store</filename> must be
+<filename><emphasis>prefix</emphasis>/store</filename> must be
 performed under the user ID that owns those directories.  This is
 typically <systemitem class="username">root</systemitem>.  (If you
 install from RPM packages, that’s in fact the default ownership.)
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index dd09e2283..039185703 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -99,7 +99,7 @@ there after an upgrade.  This means that you can <emphasis>roll
 back</emphasis> to the old version:</para>
 
 <screen>
-$ nix-env --upgrade <replaceable>some-packages</replaceable>
+$ nix-env --upgrade <emphasis>some-packages</emphasis>
 $ nix-env --rollback
 </screen>
 
diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml
index 1992c14ed..5d9a55e4e 100644
--- a/doc/manual/introduction/quick-start.xml
+++ b/doc/manual/introduction/quick-start.xml
@@ -33,7 +33,7 @@ docbook-xml-4.5
 firefox-33.0.2
 hello-2.9
 libxslt-1.1.28
-<replaceable>...</replaceable></screen>
+<emphasis>...</emphasis></screen>
 
 </para></step>
 
diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml
index 758a4500d..b8a2c7ab3 100644
--- a/doc/manual/packages/basic-package-mgmt.xml
+++ b/doc/manual/packages/basic-package-mgmt.xml
@@ -74,10 +74,10 @@ then you need to pass the path to your Nixpkgs tree using the
 <option>-f</option> flag:
 
 <screen>
-$ nix-env -qaf <replaceable>/path/to/nixpkgs</replaceable>
+$ nix-env -qaf <emphasis>/path/to/nixpkgs</emphasis>
 </screen>
 
-where <replaceable>/path/to/nixpkgs</replaceable> is where you’ve
+where <emphasis>/path/to/nixpkgs</emphasis> is where you’ve
 unpacked or checked out Nixpkgs.</para>
 
 <para>You can select specific packages by name:
diff --git a/doc/manual/packages/channels.xml b/doc/manual/packages/channels.xml
index 1ed2bba52..494a81966 100644
--- a/doc/manual/packages/channels.xml
+++ b/doc/manual/packages/channels.xml
@@ -45,7 +45,7 @@ $ nix-channel --remove nixpkgs
 $ nix-channel --update</screen>
 
 This downloads and unpacks the Nix expressions in every channel
-(downloaded from <literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal>).
+(downloaded from <literal><emphasis>url</emphasis>/nixexprs.tar.bz2</literal>).
 It also makes the union of each channel’s Nix expressions available by
 default to <command>nix-env</command> operations (via the symlink
 <filename>~/.nix-defexpr/channels</filename>).  Consequently, you can
diff --git a/doc/manual/packages/garbage-collector-roots.xml b/doc/manual/packages/garbage-collector-roots.xml
index 8338e5392..9c91862b0 100644
--- a/doc/manual/packages/garbage-collector-roots.xml
+++ b/doc/manual/packages/garbage-collector-roots.xml
@@ -8,7 +8,7 @@
 
 <para>The roots of the garbage collector are all store paths to which
 there are symlinks in the directory
-<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>.
+<filename><emphasis>prefix</emphasis>/nix/var/nix/gcroots</filename>.
 For instance, the following command makes the path
 <filename>/nix/store/d718ef...-foo</filename> a root of the collector:
 
@@ -20,7 +20,7 @@ That is, after this command, the garbage collector will not remove
 dependencies.</para>
 
 <para>Subdirectories of
-<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>
+<filename><emphasis>prefix</emphasis>/nix/var/nix/gcroots</filename>
 are also searched for symlinks.  Symlinks to non-store paths are
 followed and searched for roots, but symlinks to non-store paths
 <emphasis>inside</emphasis> the paths reached in that way are not
diff --git a/doc/manual/packages/profiles.xml b/doc/manual/packages/profiles.xml
index 2809def24..d7529d56c 100644
--- a/doc/manual/packages/profiles.xml
+++ b/doc/manual/packages/profiles.xml
@@ -118,7 +118,7 @@ can also see all available generations:
 $ nix-env --list-generations</screen></para>
 
 <para>You generally wouldn’t have
-<filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
+<filename>/nix/var/nix/profiles/<emphasis>some-profile</emphasis>/bin</filename>
 in your <literal>PATH</literal>.  Rather, there is a symlink
 <filename>~/.nix-profile</filename> that points to your current
 profile.  This means that you should put
diff --git a/doc/manual/release-notes/rl-0.10.xml b/doc/manual/release-notes/rl-0.10.xml
index 9b4233546..01f70acfd 100644
--- a/doc/manual/release-notes/rl-0.10.xml
+++ b/doc/manual/release-notes/rl-0.10.xml
@@ -46,9 +46,9 @@ irreversible.</para></warning>
       <literal>\*</literal>.</para></listitem>
 
       <listitem><para><literal>nix-env -i
-      <replaceable>pkgname</replaceable></literal> will now install
+      <emphasis>pkgname</emphasis></literal> will now install
       the highest available version of
-      <replaceable>pkgname</replaceable>, rather than installing all
+      <emphasis>pkgname</emphasis>, rather than installing all
       available versions (which would probably give collisions)
       (<literal>NIX-31</literal>).</para></listitem>
 
@@ -85,7 +85,7 @@ irreversible.</para></warning>
 "--with-freetype2-library=${freetype}/lib"</programlisting>
 
       You can write arbitrary expressions within
-      <literal>${<replaceable>...</replaceable>}</literal>, not just
+      <literal>${<emphasis>...</emphasis>}</literal>, not just
       identifiers.</para></listitem>
 
       <listitem><para>Multi-line string literals.</para></listitem>
@@ -163,7 +163,7 @@ irreversible.</para></warning>
   attribute set are unique.)  For instance, a quick way to perform a
   test build of a package in Nixpkgs is <literal>nix-build
   pkgs/top-level/all-packages.nix -A
-  <replaceable>foo</replaceable></literal>.  <literal>nix-env -q
+  <emphasis>foo</emphasis></literal>.  <literal>nix-env -q
   --attr</literal> shows the attribute names corresponding to each
   derivation.</para></listitem>
 
@@ -173,13 +173,13 @@ irreversible.</para></warning>
   <command>nix-build</command> evaluates to a function whose arguments
   all have default values, the function will be called automatically.
   Also, the new command-line switch <option>--arg
-  <replaceable>name</replaceable>
-  <replaceable>value</replaceable></option> can be used to specify
+  <emphasis>name</emphasis>
+  <emphasis>value</emphasis></option> can be used to specify
   function arguments on the command line.</para></listitem>
 
 
   <listitem><para><literal>nix-install-package --url
-  <replaceable>URL</replaceable></literal> allows a package to be
+  <emphasis>URL</emphasis></literal> allows a package to be
   installed directly from the given URL.</para></listitem>
 
 
@@ -192,7 +192,7 @@ irreversible.</para></warning>
 
 
   <listitem><para><literal>nix-build -o
-  <replaceable>symlink</replaceable></literal> allows the symlink to
+  <emphasis>symlink</emphasis></literal> allows the symlink to
   the build result to be named something other than
   <literal>result</literal>.</para></listitem>
 
diff --git a/doc/manual/release-notes/rl-0.12.xml b/doc/manual/release-notes/rl-0.12.xml
index 18978ac4b..c30124786 100644
--- a/doc/manual/release-notes/rl-0.12.xml
+++ b/doc/manual/release-notes/rl-0.12.xml
@@ -59,17 +59,17 @@ $ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</sc
 
   <listitem><para>The garbage collector has a number of new options to
   allow only some of the garbage to be deleted.  The option
-  <option>--max-freed <replaceable>N</replaceable></option> tells the
-  collector to stop after at least <replaceable>N</replaceable> bytes
+  <option>--max-freed <emphasis>N</emphasis></option> tells the
+  collector to stop after at least <emphasis>N</emphasis> bytes
   have been deleted.  The option <option>--max-links
-  <replaceable>N</replaceable></option> tells it to stop after the
+  <emphasis>N</emphasis></option> tells it to stop after the
   link count on <filename>/nix/store</filename> has dropped below
-  <replaceable>N</replaceable>.  This is useful for very large Nix
+  <emphasis>N</emphasis>.  This is useful for very large Nix
   stores on filesystems with a 32000 subdirectories limit (like
   <literal>ext3</literal>).  The option <option>--use-atime</option>
   causes store paths to be deleted in order of ascending last access
   time.  This allows non-recently used stuff to be deleted.  The
-  option <option>--max-atime <replaceable>time</replaceable></option>
+  option <option>--max-atime <emphasis>time</emphasis></option>
   specifies an upper limit to the last accessed time of paths that may
   be deleted.  For instance,
 
@@ -119,7 +119,7 @@ the following paths will be downloaded/copied (30.02 MiB):
       <listitem><para>@-patterns as in Haskell.  For instance, in a
       function definition
 
-      <programlisting>f = args @ {x, y, z}: <replaceable>...</replaceable>;</programlisting>
+      <programlisting>f = args @ {x, y, z}: <emphasis>...</emphasis>;</programlisting>
 
       <varname>args</varname> refers to the argument as a whole, which
       is further pattern-matched against the attribute set pattern
@@ -131,7 +131,7 @@ the following paths will be downloaded/copied (30.02 MiB):
       takes <emphasis>at least</emphasis> the listed attributes, while
       ignoring additional attributes.  For instance,
 
-      <programlisting>{stdenv, fetchurl, fuse, ...}: <replaceable>...</replaceable></programlisting>
+      <programlisting>{stdenv, fetchurl, fuse, ...}: <emphasis>...</emphasis></programlisting>
 
       defines a function that accepts any attribute set that includes
       at least the three listed attributes.</para></listitem>
@@ -163,7 +163,7 @@ the following paths will be downloaded/copied (30.02 MiB):
   <command>nix-pack-closure</command> and
   <command>nix-unpack-closure</command>.   You can do almost the same
   thing but much more efficiently by doing <literal>nix-store --export
-  $(nix-store -qR <replaceable>paths</replaceable>) > closure</literal> and
+  $(nix-store -qR <emphasis>paths</emphasis>) > closure</literal> and
   <literal>nix-store --import &lt;
   closure</literal>.</para></listitem>
 
diff --git a/doc/manual/release-notes/rl-0.13.xml b/doc/manual/release-notes/rl-0.13.xml
index cce2e4a26..8cb0ae9a5 100644
--- a/doc/manual/release-notes/rl-0.13.xml
+++ b/doc/manual/release-notes/rl-0.13.xml
@@ -95,9 +95,9 @@ features:</para>
 
   <listitem>
     <para>The scoping rules for <literal>inherit
-    (<replaceable>e</replaceable>) ...</literal> in recursive
+    (<emphasis>e</emphasis>) ...</literal> in recursive
     attribute sets have changed.  The expression
-    <replaceable>e</replaceable> can now refer to the attributes
+    <emphasis>e</emphasis> can now refer to the attributes
     defined in the containing set.</para>
   </listitem>
 
diff --git a/doc/manual/release-notes/rl-0.16.xml b/doc/manual/release-notes/rl-0.16.xml
index 43b7622c6..7039d4b0b 100644
--- a/doc/manual/release-notes/rl-0.16.xml
+++ b/doc/manual/release-notes/rl-0.16.xml
@@ -27,14 +27,14 @@
       2</command>), but this was not desirable because the number of
       actions to be performed in parallel was not configurable.  Nix
       now has an option <option>--cores
-      <replaceable>N</replaceable></option> as well as a configuration
+      <emphasis>N</emphasis></option> as well as a configuration
       setting <varname>build-cores =
-      <replaceable>N</replaceable></varname> that causes the
+      <emphasis>N</emphasis></varname> that causes the
       environment variable <literal>NIX_BUILD_CORES</literal> to be set to
-      <replaceable>N</replaceable> when the builder is invoked.  The
+      <emphasis>N</emphasis> when the builder is invoked.  The
       builder can use this at its discretion to perform a parallel
       build, e.g., by calling <command>make -j
-      <replaceable>N</replaceable></command>.  In Nixpkgs, this can be
+      <emphasis>N</emphasis></command>.  In Nixpkgs, this can be
       enabled on a per-package basis by setting the derivation
       attribute <varname>enableParallelBuilding</varname> to
       <literal>true</literal>.
diff --git a/doc/manual/release-notes/rl-0.6.xml b/doc/manual/release-notes/rl-0.6.xml
index 6dc6521d3..23fbee173 100644
--- a/doc/manual/release-notes/rl-0.6.xml
+++ b/doc/manual/release-notes/rl-0.6.xml
@@ -91,10 +91,10 @@
     <itemizedlist>
 
       <listitem><para>New language construct: <literal>with
-      <replaceable>E1</replaceable>;
-      <replaceable>E2</replaceable></literal> brings all attributes
-      defined in the attribute set <replaceable>E1</replaceable> in
-      scope in <replaceable>E2</replaceable>.</para></listitem>
+      <emphasis>E1</emphasis>;
+      <emphasis>E2</emphasis></literal> brings all attributes
+      defined in the attribute set <emphasis>E1</emphasis> in
+      scope in <emphasis>E2</emphasis>.</para></listitem>
 
       <listitem><para>Added a <function>map</function>
       function.</para></listitem>
diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml
index 825798fa9..1adb91a23 100644
--- a/doc/manual/release-notes/rl-0.8.xml
+++ b/doc/manual/release-notes/rl-0.8.xml
@@ -33,7 +33,7 @@ is too old (i.e., for Nix &lt;= 0.7)</screen>
 
 then you should unsubscribe from the offending channel
 (<command>nix-channel --remove
-<replaceable>URL</replaceable></command>; leave out
+<emphasis>URL</emphasis></command>; leave out
 <literal>/MANIFEST</literal>), and subscribe to the same URL, with
 <literal>channels</literal> replaced by <literal>channels-v3</literal>
 (e.g., <link
@@ -118,7 +118,7 @@ $ nix-store -q --referrers-closure \
   subscribing to a channel.</para></listitem>
 
   <listitem><para><command>nix-store -r
-  <replaceable>PATHS</replaceable></command> now builds all the
+  <emphasis>PATHS</emphasis></command> now builds all the
   derivations PATHS in parallel.  Previously it did them sequentially
   (though exploiting possible parallelism between subderivations).
   This is nice for build farms.</para></listitem>
diff --git a/doc/manual/release-notes/rl-1.1.xml b/doc/manual/release-notes/rl-1.1.xml
index 2f26e7a24..f7dadb7cb 100644
--- a/doc/manual/release-notes/rl-1.1.xml
+++ b/doc/manual/release-notes/rl-1.1.xml
@@ -72,8 +72,8 @@
     to get rid of the bootstrap binaries in the Nixpkgs source tree
     and download them instead.  You can use it by doing
     <literal>import &lt;nix/fetchurl.nix> { url =
-    <replaceable>url</replaceable>; sha256 =
-    "<replaceable>hash</replaceable>"; }</literal>. (Shea Levy)</para>
+    <emphasis>url</emphasis>; sha256 =
+    "<emphasis>hash</emphasis>"; }</literal>. (Shea Levy)</para>
   </listitem>
 
   <listitem>
diff --git a/doc/manual/release-notes/rl-1.11.xml b/doc/manual/release-notes/rl-1.11.xml
index fe422dd1f..28f6d4ac8 100644
--- a/doc/manual/release-notes/rl-1.11.xml
+++ b/doc/manual/release-notes/rl-1.11.xml
@@ -62,8 +62,8 @@ $ nix-prefetch-url -A nix-repl.src
   <listitem>
     <para>
       The new flag <option>--option build-repeat
-      <replaceable>N</replaceable></option> will cause every build to
-      be executed <replaceable>N</replaceable>+1 times. If the build
+      <emphasis>N</emphasis></option> will cause every build to
+      be executed <emphasis>N</emphasis>+1 times. If the build
       output differs between any round, the build is rejected, and the
       output paths are not registered as valid. This is primarily
       useful to verify build determinism. (We already had a
@@ -78,11 +78,11 @@ $ nix-prefetch-url -A nix-repl.src
   <listitem>
     <para>
       The options <option>--check</option> and <option>--option
-      build-repeat <replaceable>N</replaceable></option>, if they
+      build-repeat <emphasis>N</emphasis></option>, if they
       detect a difference between two runs of the same derivation and
       <option>-K</option> is given, will make the output of the other
       run available under
-      <filename><replaceable>store-path</replaceable>-check</filename>. This
+      <filename><emphasis>store-path</emphasis>-check</filename>. This
       makes it easier to investigate the non-determinism using tools
       like <command>diffoscope</command>, e.g.,
 
diff --git a/doc/manual/release-notes/rl-1.2.xml b/doc/manual/release-notes/rl-1.2.xml
index 29bd5bf81..f065f3ccc 100644
--- a/doc/manual/release-notes/rl-1.2.xml
+++ b/doc/manual/release-notes/rl-1.2.xml
@@ -56,7 +56,7 @@ outputs = [ "lib" "headers" "doc" ];
     <literal>lib</literal>, <literal>headers</literal> and
     <literal>doc</literal>.  Other packages can refer to a specific
     output by referring to
-    <literal><replaceable>pkg</replaceable>.<replaceable>output</replaceable></literal>,
+    <literal><emphasis>pkg</emphasis>.<emphasis>output</emphasis></literal>,
     e.g.
 <programlisting>
 buildInputs = [ pkg.lib pkg.headers ];
diff --git a/doc/manual/release-notes/rl-1.4.xml b/doc/manual/release-notes/rl-1.4.xml
index aefb22f2b..8114ce6b5 100644
--- a/doc/manual/release-notes/rl-1.4.xml
+++ b/doc/manual/release-notes/rl-1.4.xml
@@ -20,8 +20,8 @@ xlink:href="https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a
   <function>builtins.hashString</function>.</para></listitem>
 
   <listitem><para>Build logs are now stored in
-  <filename>/nix/var/log/nix/drvs/<replaceable>XX</replaceable>/</filename>,
-  where <replaceable>XX</replaceable> is the first two characters of
+  <filename>/nix/var/log/nix/drvs/<emphasis>XX</emphasis>/</filename>,
+  where <emphasis>XX</emphasis> is the first two characters of
   the derivation.  This is useful on machines that keep a lot of build
   logs (such as Hydra servers).</para></listitem>
 
diff --git a/doc/manual/release-notes/rl-1.6.1.xml b/doc/manual/release-notes/rl-1.6.1.xml
index 9ecc52734..982871b51 100644
--- a/doc/manual/release-notes/rl-1.6.1.xml
+++ b/doc/manual/release-notes/rl-1.6.1.xml
@@ -19,15 +19,15 @@ are:</para>
 
   <listitem>
     <para>Previously, Nix optimised expressions such as
-    <literal>"${<replaceable>expr</replaceable>}"</literal> to
-    <replaceable>expr</replaceable>.  Thus it neither checked whether
-    <replaceable>expr</replaceable> could be coerced to a string, nor
+    <literal>"${<emphasis>expr</emphasis>}"</literal> to
+    <emphasis>expr</emphasis>.  Thus it neither checked whether
+    <emphasis>expr</emphasis> could be coerced to a string, nor
     applied such coercions.  This meant that
     <literal>"${123}"</literal> evaluatued to <literal>123</literal>,
     and <literal>"${./foo}"</literal> evaluated to
     <literal>./foo</literal> (even though
     <literal>"${./foo} "</literal> evaluates to
-    <literal>"/nix/store/<replaceable>hash</replaceable>-foo "</literal>).
+    <literal>"/nix/store/<emphasis>hash</emphasis>-foo "</literal>).
     Nix now checks the type of antiquoted expressions and
     applies coercions.</para>
   </listitem>
diff --git a/doc/manual/release-notes/rl-1.7.xml b/doc/manual/release-notes/rl-1.7.xml
index 44ecaa78d..e736c15a2 100644
--- a/doc/manual/release-notes/rl-1.7.xml
+++ b/doc/manual/release-notes/rl-1.7.xml
@@ -34,7 +34,7 @@ following new features:</para>
     <command>download-via-ssh</command>, that fetches binaries from
     remote machines via SSH.  Specifying the flags <literal>--option
     use-ssh-substituter true --option ssh-substituter-hosts
-    <replaceable>user@hostname</replaceable></literal> will cause Nix
+    <emphasis>user@hostname</emphasis></literal> will cause Nix
     to download binaries from the specified machine, if it has
     them.</para>
   </listitem>
@@ -49,9 +49,9 @@ following new features:</para>
 
 <screen>
 $ nix-build '&lt;nixpkgs>' -A patchelf
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 $ nix-build '&lt;nixpkgs>' -A patchelf --check
-<replaceable>…</replaceable>
+<emphasis>…</emphasis>
 error: derivation `/nix/store/1ipvxs…-patchelf-0.6' may not be deterministic:
   hash mismatch in output `/nix/store/4pc1dm…-patchelf-0.6.drv'
 </screen>
@@ -173,11 +173,11 @@ $ nix-instantiate --eval '&lt;nixos>' -A 'config.systemd.units."nscd.service".te
   <listitem>
     <para><command>nix-collect-garbage</command> has a new flag
     <option>--delete-older-than</option>
-    <replaceable>N</replaceable><literal>d</literal>, which deletes
+    <emphasis>N</emphasis><literal>d</literal>, which deletes
     all user environment generations older than
-    <replaceable>N</replaceable> days.  Likewise, <command>nix-env
+    <emphasis>N</emphasis> days.  Likewise, <command>nix-env
     --delete-generations</command> accepts a
-    <replaceable>N</replaceable><literal>d</literal> age limit.</para>
+    <emphasis>N</emphasis><literal>d</literal> age limit.</para>
   </listitem>
 
   <listitem>
diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml
index 326990774..d7e2e99ad 100644
--- a/doc/manual/release-notes/rl-1.8.xml
+++ b/doc/manual/release-notes/rl-1.8.xml
@@ -38,8 +38,8 @@ log-servers = http://hydra.nixos.org/log
 </programlisting>
 
 then it will try to get logs from
-<literal>http://hydra.nixos.org/log/<replaceable>base name of the
-store path</replaceable></literal>. This allows you to do things like:
+<literal>http://hydra.nixos.org/log/<emphasis>base name of the
+store path</emphasis></literal>. This allows you to do things like:
 
 <screen>
 $ nix-store -l $(which xterm)
diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 57acfd0ba..6bef003ca 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -42,7 +42,7 @@
     <para>The command <command>nix-push</command> has been removed as
     part of the effort to eliminate Nix's dependency on Perl. You can
     use <command>nix copy</command> instead, e.g. <literal>nix copy
-    --to file:///tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para>
+    --to file:///tmp/my-binary-cache <emphasis>paths…</emphasis></literal></para>
   </listitem>
 
   <listitem>
@@ -136,8 +136,8 @@
         http-connections 100</literal> you can write
         <literal>--http-connections 100</literal>. Boolean options can
         be written as
-        <literal>--<replaceable>foo</replaceable></literal> or
-        <literal>--no-<replaceable>foo</replaceable></literal>
+        <literal>--<emphasis>foo</emphasis></literal> or
+        <literal>--no-<emphasis>foo</emphasis></literal>
         (e.g. <option>--no-auto-optimise-store</option>).</para>
       </listitem>
 
@@ -551,7 +551,7 @@
     is still supported for compatibility, but it is also possible to
     specify builders in <command>nix.conf</command> by setting the
     option <literal>builders =
-    @<replaceable>path</replaceable></literal>.</para>
+    @<emphasis>path</emphasis></literal>.</para>
   </listitem>
 
   <listitem>
@@ -580,9 +580,9 @@
 
   <listitem>
     <para>You can now use
-    <uri>channel:<replaceable>channel-name</replaceable></uri> as a
+    <uri>channel:<emphasis>channel-name</emphasis></uri> as a
     short-hand for
-    <uri>https://nixos.org/channels/<replaceable>channel-name</replaceable>/nixexprs.tar.xz</uri>. For
+    <uri>https://nixos.org/channels/<emphasis>channel-name</emphasis>/nixexprs.tar.xz</uri>. For
     example, <literal>nix-build channel:nixos-15.09 -A hello</literal>
     will build the GNU Hello package from the
     <literal>nixos-15.09</literal> channel. In the future, this may
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
index 3f84373e2..92abeb73e 100644
--- a/doc/manual/src/command-ref/conf-file.md
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -28,8 +28,8 @@ By default Nix reads settings from the following places:
 The configuration files consist of `name =
 value` pairs, one per line. Other files can be included with a line like
 `include
-path`, where path is interpreted relative to the current conf file and a
-missing file is an error unless `!include` is used instead. Comments
+path`, where *path* is interpreted relative to the current conf file and
+a missing file is an error unless `!include` is used instead. Comments
 start with a `#` character. Here is an example configuration file:
 
     keep-outputs = true       # Nice for developers
@@ -216,7 +216,7 @@ The following settings are currently available:
   - `hashed-mirrors`  
     A list of web servers used by `builtins.fetchurl` to obtain files by
     hash. The default is `http://tarballs.nixos.org/`. Given a hash type
-    ht and a base-16 hash h, Nix will try to download the file from
+    *ht* and a base-16 hash *h*, Nix will try to download the file from
     `hashed-mirror/ht/h`. This allows files to be downloaded even if
     they have disappeared from their original URI. For example, given
     the default mirror `http://tarballs.nixos.org/`, when building the
@@ -504,8 +504,8 @@ The following settings are currently available:
     A list of paths bind-mounted into Nix sandbox environments. You can
     use the syntax `target=source` to mount a path in a different
     location in the sandbox; for instance, `/bin=/nix-bin` will mount
-    the path `/nix-bin` as `/bin` inside the sandbox. If source is
-    followed by `?`, then it is not an error if source does not exist;
+    the path `/nix-bin` as `/bin` inside the sandbox. If *source* is
+    followed by `?`, then it is not an error if *source* does not exist;
     for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
     only be mounted in the sandbox if it exists in the host filesystem.
     
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 7d0567760..ddebc4b1b 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -43,16 +43,16 @@ paths
 # Description
 
 The `nix-build` command builds the derivations described by the Nix
-expressions in paths. If the build succeeds, it places a symlink to the
-result in the current directory. The symlink is called `result`. If
+expressions in *paths*. If the build succeeds, it places a symlink to
+the result in the current directory. The symlink is called `result`. If
 there are multiple Nix expressions, or the Nix expressions evaluate to
 multiple derivations, multiple sequentially numbered symlinks are
 created (`result`, `result-2`, and so on).
 
-If no paths are specified, then `nix-build` will use `default.nix` in
+If no *paths* are specified, then `nix-build` will use `default.nix` in
 the current directory, if it exists.
 
-If an element of paths starts with `http://` or `https://`, it is
+If an element of *paths* starts with `http://` or `https://`, it is
 interpreted as the URL of a tarball that will be downloaded and unpacked
 to a temporary location. The tarball must include a single top-level
 directory containing at least a file named `default.nix`.
@@ -83,9 +83,9 @@ All options not listed here are passed to `nix-store
   - `--dry-run`  
     Show what store paths would be built or downloaded.
 
-  - `--out-link` / `-o` outlink  
+  - `--out-link` / `-o` *outlink*  
     Change the name of the symlink to the output path created from
-    `result` to outlink.
+    `result` to *outlink*.
 
 The following common options are supported:
 
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index d427151a0..0c20788f0 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -42,25 +42,26 @@ To see the list of official NixOS channels, visit
 
 This command has the following operations:
 
-  - `--add` url \[name\]  
-    Adds a channel named name with URL url to the list of subscribed
-    channels. If name is omitted, it defaults to the last component of
-    url, with the suffixes `-stable` or `-unstable` removed.
+  - `--add` *url* \[*name*\]  
+    Adds a channel named *name* with URL *url* to the list of subscribed
+    channels. If *name* is omitted, it defaults to the last component of
+    *url*, with the suffixes `-stable` or `-unstable` removed.
 
-  - `--remove` name  
-    Removes the channel named name from the list of subscribed channels.
+  - `--remove` *name*  
+    Removes the channel named *name* from the list of subscribed
+    channels.
 
   - `--list`  
     Prints the names and URLs of all subscribed channels on standard
     output.
 
-  - `--update` \[names…\]  
+  - `--update` \[*names*…\]  
     Downloads the Nix expressions of all subscribed channels (or only
-    those included in names if specified) and makes them the default for
-    `nix-env` operations (by symlinking them from the directory
+    those included in *names* if specified) and makes them the default
+    for `nix-env` operations (by symlinking them from the directory
     `~/.nix-defexpr`).
 
-  - `--rollback` \[generation\]  
+  - `--rollback` \[*generation*\]  
     Reverts the previous call to `nix-channel
                     --update`. Optionally, you can specify a specific channel generation
     number to restore.
diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index 7946dc875..77b5d42d3 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -33,7 +33,7 @@ additional options: `-d` (`--delete-old`), which deletes all old
 generations of all profiles in `/nix/var/nix/profiles` by invoking
 `nix-env --delete-generations old` on all profiles (of course, this
 makes rollbacks to previous configurations impossible); and
-`--delete-older-than` period, where period is a value such as `30d`,
+`--delete-older-than` *period*, where period is a value such as `30d`,
 which deletes all generations older than the specified number of days in
 all profiles in `/nix/var/nix/profiles` (except for the generations that
 were active at that point in time).
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index 90bf6ea08..90a2bd351 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -98,7 +98,7 @@ This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
 have an effect. See also [???](#sec-common-options).
 
-  - `--file` / `-f` path  
+  - `--file` / `-f` *path*  
     Specifies the Nix expression (designated below as the *active Nix
     expression*) used by the `--install`, `--upgrade`, and `--query
     --available` operations to obtain derivations. The default is
@@ -109,7 +109,7 @@ have an effect. See also [???](#sec-common-options).
     unpacked to a temporary location. The tarball must include a single
     top-level directory containing at least a file named `default.nix`.
 
-  - `--profile` / `-p` path  
+  - `--profile` / `-p` *path*  
     Specifies the profile to be used by those operations that operate on
     a profile (designated below as the *active profile*). A profile is a
     sequence of user environments called *generations*, one of which is
@@ -125,10 +125,10 @@ have an effect. See also [???](#sec-common-options).
     [substituted](#gloss-substitute) (i.e., downloaded) and which paths
     will be built from source (because no substitute is available).
 
-  - `--system-filter` system  
+  - `--system-filter` *system*  
     By default, operations such as `--query
                     --available` show derivations matching any platform. This option
-    allows you to use derivations for the specified platform system.
+    allows you to use derivations for the specified platform *system*.
 
 <!-- end list -->
 
@@ -200,17 +200,17 @@ args
 
 The install operation creates a new user environment, based on the
 current generation of the active profile, to which a set of store paths
-described by args is added. The arguments args map to store paths in a
-number of possible ways:
+described by *args* is added. The arguments *args* map to store paths in
+a number of possible ways:
 
-  - By default, args is a set of derivation names denoting derivations
+  - By default, *args* is a set of derivation names denoting derivations
     in the active Nix expression. These are realised, and the resulting
     output paths are installed. Currently installed derivations with a
     name equal to the name of a derivation being added are removed
     unless the option `--preserve-installed` is specified.
     
-    If there are multiple derivations matching a name in args that have
-    the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the
+    If there are multiple derivations matching a name in *args* that
+    have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the
     derivation with the highest *priority* is used. A derivation can
     define a priority by declaring the `meta.priority` attribute. This
     attribute should be a number, with a higher value denoting a lower
@@ -230,22 +230,23 @@ number of possible ways:
     unambiguous. To find out the attribute paths of available packages,
     use `nix-env -qaP`.
 
-  - If `--from-profile` path is given, args is a set of names denoting
-    installed store paths in the profile path. This is an easy way to
-    copy user environment elements from one profile to another.
+  - If `--from-profile` *path* is given, *args* is a set of names
+    denoting installed store paths in the profile *path*. This is an
+    easy way to copy user environment elements from one profile to
+    another.
 
-  - If `--from-expression` is given, args are Nix
+  - If `--from-expression` is given, *args* are Nix
     [functions](#ss-functions) that are called with the active Nix
     expression as their single argument. The derivations returned by
     those function calls are installed. This allows derivations to be
     specified in an unambiguous way, which is necessary if there are
     multiple derivations with the same name.
 
-  - If args are store derivations, then these are
+  - If *args* are store derivations, then these are
     [realised](#rsec-nix-store-realise), and the resulting output paths
     are installed.
 
-  - If args are store paths that are not store derivations, then these
+  - If *args* are store paths that are not store derivations, then these
     are [realised](#rsec-nix-store-realise) and installed.
 
   - By default all outputs are installed for each derivation. That can
@@ -359,12 +360,12 @@ args
 The upgrade operation creates a new user environment, based on the
 current generation of the active profile, in which all store paths are
 replaced for which there are newer versions in the set of paths
-described by args. Paths for which there are no newer versions are left
-untouched; this is not an error. It is also not an error if an element
-of args matches no installed derivations.
+described by *args*. Paths for which there are no newer versions are
+left untouched; this is not an error. It is also not an error if an
+element of *args* matches no installed derivations.
 
-For a description of how args is mapped to a set of store paths, see
-[`--install`](#rsec-nix-env-install). If args describes multiple store
+For a description of how *args* is mapped to a set of store paths, see
+[`--install`](#rsec-nix-env-install). If *args* describes multiple store
 paths with the same symbolic name, only the one with the highest version
 is installed.
 
@@ -462,7 +463,7 @@ drvnames
 
 The uninstall operation creates a new user environment, based on the
 current generation of the active profile, from which the store paths
-designated by the symbolic names names are removed.
+designated by the symbolic names *names* are removed.
 
 ## Examples
 
@@ -629,7 +630,7 @@ The query operation displays information about either the store paths
 that are installed in the current generation of the active profile
 (`--installed`), or the derivations that are available for installation
 in the active Nix expression (`--available`). It only prints information
-about derivations whose symbolic name matches one of names.
+about derivations whose symbolic name matches one of *names*.
 
 The derivations are sorted by their `name` attributes.
 
@@ -696,14 +697,14 @@ derivation is shown unless `--no-name` is specified.
     upgrades for installed packages are available in a Nix expression. A
     column is added with the following meaning:
     
-      - `<` version  
+      - `<` *version*  
         A newer version of the package is available or installed.
     
-      - `=` version  
+      - `=` *version*  
         At most the same version of the package is available or
         installed.
     
-      - `>` version  
+      - `>` *version*  
         Only older versions of the package are available or installed.
     
       - `- ?`  
@@ -806,8 +807,8 @@ path
 
 ## Description
 
-This operation makes path the current profile for the user. That is, the
-symlink `~/.nix-profile` is made to point to path.
+This operation makes *path* the current profile for the user. That is,
+the symlink `~/.nix-profile` is made to point to *path*.
 
 ## Examples
 
@@ -882,9 +883,9 @@ generation
 
 ## Description
 
-This operation makes generation number generation the current generation
-of the active profile. That is, if the `profile` is the path to the
-active profile, then the symlink `profile` is made to point to
+This operation makes generation number *generation* the current
+generation of the active profile. That is, if the `profile` is the path
+to the active profile, then the symlink `profile` is made to point to
 `profile-generation-link`, which is in turn a symlink to the actual user
 environment in the Nix store.
 
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index d433cbc5b..3b8bbf740 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -37,7 +37,7 @@ hash
 # Description
 
 The command `nix-hash` computes the cryptographic hash of the contents
-of each path and prints it on standard output. By default, it computes
+of each *path* and prints it on standard output. By default, it computes
 an MD5 hash, but other hash algorithms are available as well. The hash
 is printed in hexadecimal. To generate the same hash as
 `nix-prefetch-url` you have to specify multiple arguments, see below for
@@ -55,9 +55,9 @@ path | md5sum`.
 
   - `--flat`  
     Print the cryptographic hash of the contents of each regular file
-    path. That is, do not compute the hash over the dump of path. The
-    result is identical to that produced by the GNU commands `md5sum`
-    and `sha1sum`.
+    *path*. That is, do not compute the hash over the dump of *path*.
+    The result is identical to that produced by the GNU commands
+    `md5sum` and `sha1sum`.
 
   - `--base32`  
     Print the hash in a base-32 representation rather than hexadecimal.
@@ -67,17 +67,17 @@ path | md5sum`.
   - `--truncate`  
     Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits.
 
-  - `--type` hashAlgo  
+  - `--type` *hashAlgo*  
     Use the specified cryptographic hash algorithm, which can be one of
     `md5`, `sha1`, and `sha256`.
 
   - `--to-base16`  
     Don’t hash anything, but convert the base-32 hash representation
-    hash to hexadecimal.
+    *hash* to hexadecimal.
 
   - `--to-base32`  
     Don’t hash anything, but convert the hexadecimal hash representation
-    hash to base-32.
+    *hash* to base-32.
 
 # Examples
 
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 449f5f70f..179fbf5ba 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -56,19 +56,19 @@ files
 
 The command `nix-instantiate` generates [store
 derivations](#gloss-derivation) from (high-level) Nix expressions. It
-evaluates the Nix expressions in each of files (which defaults to
-./default.nix). Each top-level expression should evaluate to a
+evaluates the Nix expressions in each of *files* (which defaults to
+*./default.nix*). Each top-level expression should evaluate to a
 derivation, a list of derivations, or a set of derivations. The paths of
 the resulting store derivations are printed on standard output.
 
-If files is the character `-`, then a Nix expression will be read from
+If *files* is the character `-`, then a Nix expression will be read from
 standard input.
 
 See also [???](#sec-common-options) for a list of common options.
 
 # Options
 
-  - `--add-root` path; `--indirect`  
+  - `--add-root` *path*; `--indirect`  
     See the [corresponding options](#opt-add-root) in `nix-store`.
 
   - `--parse`  
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index bf8084e45..f7de27402 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -31,9 +31,9 @@ hash
 # Description
 
 The command `nix-prefetch-url` downloads the file referenced by the URL
-url, prints its cryptographic hash, and copies it into the Nix store.
-The file name in the store is `hash-baseName`, where baseName is
-everything following the final slash in url.
+*url*, prints its cryptographic hash, and copies it into the Nix store.
+The file name in the store is `hash-baseName`, where *baseName* is
+everything following the final slash in *url*.
 
 This command is just a convenience for Nix expression writers. Often a
 Nix expression fetches some source distribution from the network using
@@ -44,10 +44,10 @@ again when you build your Nix expression. Since `fetchurl` uses the same
 name for the downloaded file as `nix-prefetch-url`, the redundant
 download can be avoided.
 
-If hash is specified, then a download is not performed if the Nix store
-already contains a file with the same hash and base name. Otherwise, the
-file is downloaded, and an error is signaled if the actual hash of the
-file does not match the specified hash.
+If *hash* is specified, then a download is not performed if the Nix
+store already contains a file with the same hash and base name.
+Otherwise, the file is downloaded, and an error is signaled if the
+actual hash of the file does not match the specified hash.
 
 This command prints the hash on standard output. Additionally, if the
 option `--print-path` is used, the path of the downloaded file in the
@@ -55,7 +55,7 @@ Nix store is also printed.
 
 # Options
 
-  - `--type` hashAlgo  
+  - `--type` *hashAlgo*  
     Use the specified cryptographic hash algorithm, which can be one of
     `md5`, `sha1`, and `sha256`.
 
@@ -67,10 +67,10 @@ Nix store is also printed.
     result to the Nix store. The resulting hash can be used with
     functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`.
 
-  - `--name` name  
+  - `--name` *name*  
     Override the name of the file in the Nix store. By default, this is
-    `hash-basename`, where basename is the last component of url.
-    Overriding the name is necessary when basename contains characters
+    `hash-basename`, where *basename* is the last component of *url*.
+    Overriding the name is necessary when *basename* contains characters
     that are not allowed in Nix store paths.
 
 # Examples
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index c6910e3f5..9e2b781ab 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -61,14 +61,14 @@ path
 The command `nix-shell` will build the dependencies of the specified
 derivation, but not the derivation itself. It will then start an
 interactive shell in which all environment variables defined by the
-derivation path have been set to their corresponding values, and the
+derivation *path* have been set to their corresponding values, and the
 script `$stdenv/setup` has been sourced. This is useful for reproducing
 the environment of a derivation for development.
 
-If path is not given, `nix-shell` defaults to `shell.nix` if it exists,
-and `default.nix` otherwise.
+If *path* is not given, `nix-shell` defaults to `shell.nix` if it
+exists, and `default.nix` otherwise.
 
-If path starts with `http://` or `https://`, it is interpreted as the
+If *path* starts with `http://` or `https://`, it is interpreted as the
 URL of a tarball that will be downloaded and unpacked to a temporary
 location. The tarball must include a single top-level directory
 containing at least a file named `default.nix`.
@@ -91,8 +91,8 @@ All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
 `nix-instantiate`. See also [???](#sec-common-options).
 
-  - `--command` cmd  
-    In the environment of the derivation, run the shell command cmd.
+  - `--command` *cmd*  
+    In the environment of the derivation, run the shell command *cmd*.
     This command is executed in an interactive shell. (Use `--run` to
     use a non-interactive shell instead.) However, a call to `exit` is
     implicitly added to the command, so the shell will exit after
@@ -102,14 +102,14 @@ All options not listed here are passed to `nix-store
     interactive shell. This can be useful for doing any additional
     initialisation.
 
-  - `--run` cmd  
+  - `--run` *cmd*  
     Like `--command`, but executes the command in a non-interactive
     shell. This means (among other things) that if you hit Ctrl-C while
     the command is running, the shell exits.
 
-  - `--exclude` regexp  
+  - `--exclude` *regexp*  
     Do not build any dependencies whose store path matches the regular
-    expression regexp. This option may be specified multiple times.
+    expression *regexp*. This option may be specified multiple times.
 
   - `--pure`  
     If this flag is specified, the environment is almost entirely
@@ -120,19 +120,19 @@ All options not listed here are passed to `nix-store
     installation) `/etc/bashrc` are still sourced, so any variables set
     there will affect the interactive shell.
 
-  - `--packages` / `-p` packages…  
+  - `--packages` / `-p` *packages*…  
     Set up an environment in which the specified packages are present.
     The command line arguments are interpreted as attribute names inside
     the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk`
     will start a shell in which the packages denoted by the attribute
     names `libjpeg` and `openjdk` are present.
 
-  - `-i` interpreter  
+  - `-i` *interpreter*  
     The chained script interpreter to be invoked by `nix-shell`. Only
     applicable in `#!`-scripts (described
     [below](#ssec-nix-shell-shebang)).
 
-  - `--keep` name  
+  - `--keep` *name*  
     When a `--pure` shell is started, keep the listed environment
     variables.
 
@@ -199,10 +199,10 @@ done by starting the script with the following lines:
     #! /usr/bin/env nix-shell
     #! nix-shell -i real-interpreter -p packages
 
-where real-interpreter is the “real” script interpreter that will be
+where *real-interpreter* is the “real” script interpreter that will be
 invoked by `nix-shell` after it has obtained the dependencies and
-initialised the environment, and packages are the attribute names of the
-dependencies in Nixpkgs.
+initialised the environment, and *packages* are the attribute names of
+the dependencies in Nixpkgs.
 
 The lines starting with `#! nix-shell` specify `nix-shell` options (see
 above). Note that you cannot write `#! /usr/bin/env nix-shell -i ...`
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 703e71076..a666de49b 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -37,10 +37,10 @@ options are allowed for every subcommand, though they may not always
 have an effect. See also [???](#sec-common-options) for a list of common
 options.
 
-  - `--add-root` path  
+  - `--add-root` *path*  
     Causes the result of a realisation (`--realise` and
     `--force-realise`) to be registered as a root of the garbage
-    collector(see [???](#ssec-gc-roots)). The root is stored in path,
+    collector(see [???](#ssec-gc-roots)). The root is stored in *path*,
     which must be inside a directory that is scanned for roots by the
     garbage collector (i.e., typically in a subdirectory of
     `/nix/var/nix/gcroots/`) *unless* the `--indirect` flag is used.
@@ -56,8 +56,8 @@ options.
     result in the current directory; such a build result should not be
     garbage-collected unless the symlink is removed.
     
-    The `--indirect` flag causes a uniquely named symlink to path to be
-    stored in `/nix/var/nix/gcroots/auto/`. For instance,
+    The `--indirect` flag causes a uniquely named symlink to *path* to
+    be stored in `/nix/var/nix/gcroots/auto/`. For instance,
     
         $ nix-store --add-root /home/eelco/bla/result --indirect -r ...
         
@@ -262,10 +262,11 @@ The following suboperations may be specified:
 By default, all unreachable paths are deleted. The following options
 control what gets deleted and in what order:
 
-  - `--max-freed` bytes  
-    Keep deleting paths until at least bytes bytes have been deleted,
-    then stop. The argument bytes can be followed by the multiplicative
-    suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB or TiB units.
+  - `--max-freed` *bytes*  
+    Keep deleting paths until at least *bytes* bytes have been deleted,
+    then stop. The argument *bytes* can be followed by the
+    multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB
+    or TiB units.
 
 The behaviour of the collector is also influenced by the
 [`keep-outputs`](#conf-keep-outputs) and
@@ -303,7 +304,7 @@ paths
 
 ## Description
 
-The operation `--delete` deletes the store paths paths from the Nix
+The operation `--delete` deletes the store paths *paths* from the Nix
 store, but only if it is safe to do so; that is, when the path is not
 reachable from a root of the garbage collector. This means that you can
 only delete paths that would also be deleted by `nix-store --gc`. Thus,
@@ -379,7 +380,7 @@ The operation `--query` displays various bits of information about the
 store paths . The queries are described below. At most one query can be
 specified. The default query is `--outputs`.
 
-The paths paths may also be symlinks from outside of the Nix store, to
+The paths *paths* may also be symlinks from outside of the Nix store, to
 the Nix store. In that case, the query is applied to the target of the
 symlink.
 
@@ -397,11 +398,11 @@ symlink.
 
   - `--outputs`  
     Prints out the [output paths](#gloss-output-path) of the store
-    derivations paths. These are the paths that will be produced when
+    derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
   - `--requisites`; `-R`  
-    Prints out the [closure](#gloss-closure) of the store path paths.
+    Prints out the [closure](#gloss-closure) of the store path *paths*.
     
     This query has one option:
     
@@ -419,29 +420,30 @@ symlink.
 
   - `--references`  
     Prints the set of [references](#gloss-reference) of the store paths
-    paths, that is, their immediate dependencies. (For *all*
+    *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
   - `--referrers`  
-    Prints the set of *referrers* of the store paths paths, that is, the
-    store paths currently existing in the Nix store that refer to one of
-    paths. Note that contrary to the references, the set of referrers is
-    not constant; it can change as store paths are added or removed.
+    Prints the set of *referrers* of the store paths *paths*, that is,
+    the store paths currently existing in the Nix store that refer to
+    one of *paths*. Note that contrary to the references, the set of
+    referrers is not constant; it can change as store paths are added or
+    removed.
 
   - `--referrers-closure`  
-    Prints the closure of the set of store paths paths under the
+    Prints the closure of the set of store paths *paths* under the
     referrers relation; that is, all store paths that directly or
-    indirectly refer to one of paths. These are all the path currently
-    in the Nix store that are dependent on paths.
+    indirectly refer to one of *paths*. These are all the path currently
+    in the Nix store that are dependent on *paths*.
 
   - `--deriver`; `-d`  
-    Prints the [deriver](#gloss-deriver) of the store paths paths. If
+    Prints the [deriver](#gloss-deriver) of the store paths *paths*. If
     the path has no deriver (e.g., if it is a source file), or if the
     deriver is not known (e.g., in the case of a binary-only
     deployment), the string `unknown-deriver` is printed.
 
   - `--graph`  
-    Prints the references graph of the store paths paths in the format
+    Prints the references graph of the store paths *paths* in the format
     of the `dot` tool of AT\&T's [Graphviz
     package](http://www.graphviz.org/). This can be used to visualise
     dependency graphs. To obtain a build-time dependency graph, apply
@@ -449,40 +451,40 @@ symlink.
     apply it to an output path.
 
   - `--tree`  
-    Prints the references graph of the store paths paths as a nested
+    Prints the references graph of the store paths *paths* as a nested
     ASCII tree. References are ordered by descending closure size; this
     tends to flatten the tree, making it more readable. The query only
     recurses into a store path when it is first encountered; this
     prevents a blowup of the tree representation of the graph.
 
   - `--graphml`  
-    Prints the references graph of the store paths paths in the
+    Prints the references graph of the store paths *paths* in the
     [GraphML](http://graphml.graphdrawing.org/) file format. This can be
     used to visualise dependency graphs. To obtain a build-time
     dependency graph, apply this to a store derivation. To obtain a
     runtime dependency graph, apply it to an output path.
 
-  - `--binding` name; `-b` name  
-    Prints the value of the attribute name (i.e., environment variable)
-    of the store derivations paths. It is an error for a derivation to
-    not have the specified attribute.
+  - `--binding` *name*; `-b` *name*  
+    Prints the value of the attribute *name* (i.e., environment
+    variable) of the store derivations *paths*. It is an error for a
+    derivation to not have the specified attribute.
 
   - `--hash`  
-    Prints the SHA-256 hash of the contents of the store paths paths
+    Prints the SHA-256 hash of the contents of the store paths *paths*
     (that is, the hash of the output of `nix-store --dump` on the given
     paths). Since the hash is stored in the Nix database, this is a fast
     operation.
 
   - `--size`  
-    Prints the size in bytes of the contents of the store paths paths —
-    to be precise, the size of the output of `nix-store --dump` on the
-    given paths. Note that the actual disk space required by the store
-    paths may be higher, especially on filesystems with large cluster
-    sizes.
+    Prints the size in bytes of the contents of the store paths *paths*
+    — to be precise, the size of the output of `nix-store --dump` on
+    the given paths. Note that the actual disk space required by the
+    store paths may be higher, especially on filesystems with large
+    cluster sizes.
 
   - `--roots`  
     Prints the garbage collector roots that point, directly or
-    indirectly, at the store paths paths.
+    indirectly, at the store paths *paths*.
 
 ## Examples
 
@@ -708,8 +710,8 @@ path
 ## Description
 
 The operation `--dump` produces a NAR (Nix ARchive) file containing the
-contents of the file system tree rooted at path. The archive is written
-to standard output.
+contents of the file system tree rooted at *path*. The archive is
+written to standard output.
 
 A NAR archive is like a TAR or Zip archive, but it contains only the
 information that Nix considers important. For instance, timestamps are
@@ -745,8 +747,8 @@ path
 
 ## Description
 
-The operation `--restore` unpacks a NAR archive to path, which must not
-already exist. The archive is read from standard input.
+The operation `--restore` unpacks a NAR archive to *path*, which must
+not already exist. The archive is read from standard input.
 
 # Operation `--export`
 
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index ac9d996c3..dce95773a 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -44,9 +44,9 @@ Most Nix commands accept the following command-line options:
     This option may be specified repeatedly. See the previous verbosity
     levels list.
 
-  - `--log-format` format  
+  - `--log-format` *format*  
     This option can be used to change the output of the log format, with
-    format being one of:
+    *format* being one of:
     
       - raw  
         This is the raw format, as outputted by nix-build.
@@ -68,7 +68,7 @@ Most Nix commands accept the following command-line options:
     output and error are always written to a log file in
     `prefix/nix/var/log/nix`.
 
-  - `--max-jobs` / `-j` number  
+  - `--max-jobs` / `-j` *number*  
     Sets the maximum number of build jobs that Nix will perform in
     parallel to the specified number. Specify `auto` to use the number
     of CPUs in the system. The default is specified by the
@@ -144,7 +144,7 @@ Most Nix commands accept the following command-line options:
     database. Most Nix operations do need database access, so those
     operations will fail.
 
-  - `--arg` name value  
+  - `--arg` *name* *value*  
     This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell`
     and `nix-build`. When evaluating Nix expressions, the expression
     evaluator will automatically try to call functions that it
@@ -153,8 +153,8 @@ Most Nix commands accept the following command-line options:
             defaultValue }:
             ...`). With `--arg`, you can also call functions that have arguments
     without a default value (or override a default value). That is, if
-    the evaluator encounters a function with an argument named name, it
-    will call it with value value.
+    the evaluator encounters a function with an argument named *name*,
+    it will call it with value *value*.
     
     For instance, the top-level `default.nix` in Nixpkgs is actually a
     function:
@@ -172,18 +172,18 @@ Most Nix commands accept the following command-line options:
             \"i686-freebsd\"`. (Note that since the argument is a Nix string
     literal, you have to escape the quotes.)
 
-  - `--argstr` name value  
+  - `--argstr` *name* *value*  
     This option is like `--arg`, only the value is not a Nix expression
     but a string. So instead of `--arg system \"i686-linux\"` (the outer
     quotes are to keep the shell happy) you can say `--argstr system
             i686-linux`.
 
-  - `--attr` / `-A` attrPath  
+  - `--attr` / `-A` *attrPath*  
     Select an attribute from the top-level Nix expression being
     evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and
-    `nix-shell` only.) The *attribute path* attrPath is a sequence of
+    `nix-shell` only.) The *attribute path* *attrPath* is a sequence of
     attribute names separated by dots. For instance, given a top-level
-    Nix expression e, the attribute path `xorg.xorgserver` would cause
+    Nix expression *e*, the attribute path `xorg.xorgserver` would cause
     the expression `e.xorg.xorgserver` to be used. See [`nix-env
             --install`](#refsec-nix-env-install-examples) for some concrete
     examples.
@@ -204,14 +204,14 @@ Most Nix commands accept the following command-line options:
     use, give your expression to the `nix-shell -p` convenience flag
     instead.
 
-  - `-I` path  
+  - `-I` *path*  
     Add a path to the Nix expression search path. This option may be
     given multiple times. See the NIX\_PATH\</literal\> environment
     variable for information on the semantics of the Nix search path.
     Paths added through `-I` take precedence over `NIX_PATH`.
 
-  - `--option` name value  
-    Set the Nix configuration option name to value. This overrides
+  - `--option` *name* *value*  
+    Set the Nix configuration option *name* to *value*. This overrides
     settings in the Nix configuration file (see nix.conf5).
 
   - `--repair`  
diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
index 01e18513d..3582fa268 100644
--- a/doc/manual/src/expressions/advanced-attributes.md
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -54,9 +54,9 @@ Derivations can declare some infrequently used optional attributes.
     attribute should be a list of pairs `[ name1
                     path1 name2
                     path2 ...
-                    ]`. The references graph of each pathN will be stored in a text file
-    nameN in the temporary build directory. The text files have the
-    format used by `nix-store
+                    ]`. The references graph of each *pathN* will be stored in a text
+    file *nameN* in the temporary build directory. The text files have
+    the format used by `nix-store
                     --register-validity` (with the deriver fields left empty). For
     example, when the following derivation is built:
     
@@ -204,9 +204,9 @@ Derivations can declare some infrequently used optional attributes.
     then when the builder runs, the environment variable `bigPath` will
     contain the absolute path to a temporary file containing `a very
     long
-                    string`. That is, for any attribute x listed in `passAsFile`, Nix
+                    string`. That is, for any attribute *x* listed in `passAsFile`, Nix
     will pass an environment variable `xPath` holding the path of the
-    file containing the value of attribute x. This is useful when you
+    file containing the value of attribute *x*. This is useful when you
     need to pass large strings to a builder, since most operating
     systems impose a limit on the size of the environment (typically, a
     few hundred kilobyte).
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index 8faf4b939..8f19c692a 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -9,42 +9,42 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `abort` s; `builtins.abort` s  
-    Abort Nix expression evaluation, print error message s.
+  - `abort` *s*; `builtins.abort` *s*  
+    Abort Nix expression evaluation, print error message *s*.
 
-  - `builtins.add` e1 e2  
-    Return the sum of the numbers e1 and e2.
+  - `builtins.add` *e1* *e2*  
+    Return the sum of the numbers *e1* and *e2*.
 
-  - `builtins.all` pred list  
-    Return `true` if the function pred returns `true` for all elements
-    of list, and `false` otherwise.
+  - `builtins.all` *pred* *list*  
+    Return `true` if the function *pred* returns `true` for all elements
+    of *list*, and `false` otherwise.
 
-  - `builtins.any` pred list  
-    Return `true` if the function pred returns `true` for at least one
-    element of list, and `false` otherwise.
+  - `builtins.any` *pred* *list*  
+    Return `true` if the function *pred* returns `true` for at least one
+    element of *list*, and `false` otherwise.
 
-  - `builtins.attrNames` set  
-    Return the names of the attributes in the set set in an
+  - `builtins.attrNames` *set*  
+    Return the names of the attributes in the set *set* in an
     alphabetically sorted list. For instance, `builtins.attrNames { y
     = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`.
 
-  - `builtins.attrValues` set  
-    Return the values of the attributes in the set set in the order
+  - `builtins.attrValues` *set*  
+    Return the values of the attributes in the set *set* in the order
     corresponding to the sorted attribute names.
 
-  - `baseNameOf` s  
-    Return the *base name* of the string s, that is, everything
+  - `baseNameOf` *s*  
+    Return the *base name* of the string *s*, that is, everything
     following the final slash in the string. This is similar to the GNU
     `basename` command.
 
-  - `builtins.bitAnd` e1 e2  
-    Return the bitwise AND of the integers e1 and e2.
+  - `builtins.bitAnd` *e1* *e2*  
+    Return the bitwise AND of the integers *e1* and *e2*.
 
-  - `builtins.bitOr` e1 e2  
-    Return the bitwise OR of the integers e1 and e2.
+  - `builtins.bitOr` *e1* *e2*  
+    Return the bitwise OR of the integers *e1* and *e2*.
 
-  - `builtins.bitXor` e1 e2  
-    Return the bitwise XOR of the integers e1 and e2.
+  - `builtins.bitXor` *e1* *e2*  
+    Return the bitwise XOR of the integers *e1* and *e2*.
 
   - `builtins`  
     The set `builtins` contains all the built-in functions and values.
@@ -56,17 +56,17 @@ For instance, `derivation` is also available as `builtins.derivation`.
     This allows a Nix expression to fall back gracefully on older Nix
     installations that don’t have the desired built-in function.
 
-  - `builtins.compareVersions` s1 s2  
+  - `builtins.compareVersions` *s1* *s2*  
     Compare two strings representing versions and return `-1` if version
-    s1 is older than version s2, `0` if they are the same, and `1` if s1
-    is newer than s2. The version comparison algorithm is the same as
-    the one used by [`nix-env
+    *s1* is older than version *s2*, `0` if they are the same, and `1`
+    if *s1* is newer than *s2*. The version comparison algorithm is the
+    same as the one used by [`nix-env
                     -u`](#ssec-version-comparisons).
 
-  - `builtins.concatLists` lists  
+  - `builtins.concatLists` *lists*  
     Concatenate a list of lists into a single list.
 
-  - `builtins.concatStringsSep` separator list  
+  - `builtins.concatStringsSep` *separator* *list*  
     Concatenate a list of strings with a separator between each element,
     e.g. `concatStringsSep "/"
                     ["usr" "local" "bin"] == "usr/local/bin"`
@@ -76,37 +76,37 @@ For instance, `derivation` is also available as `builtins.derivation`.
     identifier for the Nix installation on which the expression is being
     evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
 
-  - `builtins.deepSeq` e1 e2  
+  - `builtins.deepSeq` *e1* *e2*  
     This is like `seq
                     e1
-                    e2`, except that e1 is evaluated *deeply*: if it’s a list or set,
+                    e2`, except that *e1* is evaluated *deeply*: if it’s a list or set,
     its elements or attributes are also evaluated recursively.
 
-  - `derivation` attrs; `builtins.derivation` attrs  
+  - `derivation` *attrs*; `builtins.derivation` *attrs*  
     `derivation` is described in [???](#ssec-derivation).
 
-  - `dirOf` s; `builtins.dirOf` s  
-    Return the directory part of the string s, that is, everything
+  - `dirOf` *s*; `builtins.dirOf` *s*  
+    Return the directory part of the string *s*, that is, everything
     before the final slash in the string. This is similar to the GNU
     `dirname` command.
 
-  - `builtins.div` e1 e2  
-    Return the quotient of the numbers e1 and e2.
+  - `builtins.div` *e1* *e2*  
+    Return the quotient of the numbers *e1* and *e2*.
 
-  - `builtins.elem` x xs  
-    Return `true` if a value equal to x occurs in the list xs, and
+  - `builtins.elem` *x* *xs*  
+    Return `true` if a value equal to *x* occurs in the list *xs*, and
     `false` otherwise.
 
-  - `builtins.elemAt` xs n  
-    Return element n from the list xs. Elements are counted starting
+  - `builtins.elemAt` *xs* *n*  
+    Return element *n* from the list *xs*. Elements are counted starting
     from 0. A fatal error occurs if the index is out of bounds.
 
-  - `builtins.fetchurl` url  
+  - `builtins.fetchurl` *url*  
     Download the specified URL and return the path of the downloaded
     file. This function is not available if [restricted evaluation
     mode](#conf-restrict-eval) is enabled.
 
-  - `fetchTarball` url; `builtins.fetchTarball` url  
+  - `fetchTarball` *url*; `builtins.fetchTarball` *url*  
     Download the specified URL, unpack it and return the path of the
     unpacked tree. The file must be a tape archive (`.tar`) compressed
     with `gzip`, `bzip2` or `xz`. The top-level path component of the
@@ -142,10 +142,10 @@ For instance, `derivation` is also available as `builtins.derivation`.
     This function is not available if [restricted evaluation
     mode](#conf-restrict-eval) is enabled.
 
-  - `builtins.fetchGit` args  
-    Fetch a path from git. args can be a URL, in which case the HEAD of
-    the repo at that URL is fetched. Otherwise, it can be an attribute
-    with the following attributes (all except `url` optional):
+  - `builtins.fetchGit` *args*  
+    Fetch a path from git. *args* can be a URL, in which case the HEAD
+    of the repo at that URL is fetched. Otherwise, it can be an
+    attribute with the following attributes (all except `url` optional):
     
       - url  
         The URL of the repo.
@@ -240,11 +240,11 @@ For instance, `derivation` is also available as `builtins.derivation`.
         > 
         > This behavior is disabled in *Pure evaluation mode*.
 
-  - `builtins.filter` f xs  
-    Return a list consisting of the elements of xs for which the
-    function f returns `true`.
+  - `builtins.filter` *f* *xs*  
+    Return a list consisting of the elements of *xs* for which the
+    function *f* returns `true`.
 
-  - `builtins.filterSource` e1 e2  
+  - `builtins.filterSource` *e1* *e2*  
     This function allows you to copy sources into the Nix store while
     filtering certain files. For instance, suppose that you want to use
     the directory `source-dir` as an input to a Nix expression, e.g.
@@ -266,9 +266,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
         ./source-dir;
     ```
     
-    Thus, the first argument e1 must be a predicate function that is
+    Thus, the first argument *e1* must be a predicate function that is
     called for each regular file, directory or symlink in the source
-    tree e2. If the function returns `true`, the file is copied to the
+    tree *e2*. If the function returns `true`, the file is copied to the
     Nix store, otherwise it is omitted. The function is called with two
     arguments. The first is the full path of the file. The second is a
     string that identifies the type of the file, which is either
@@ -276,19 +276,19 @@ For instance, `derivation` is also available as `builtins.derivation`.
     kinds of files such as device nodes or fifos — but note that those
     cannot be copied to the Nix store, so if the predicate returns
     `true` for them, the copy will fail). If you exclude a directory,
-    the entire corresponding subtree of e2 will be excluded.
+    the entire corresponding subtree of *e2* will be excluded.
 
-  - `builtins.foldl’` op nul list  
+  - `builtins.foldl’` *op* *nul* *list*  
     Reduce a list by applying a binary operator, from left to right,
     e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op
                     (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e.,
     its arguments are evaluated first. For example, `foldl’ (x: y: x +
     y) 0 [1 2 3]` evaluates to 6.
 
-  - `builtins.functionArgs` f  
+  - `builtins.functionArgs` *f*  
     Return a set containing the names of the formal arguments expected
-    by the function f. The value of each attribute is a Boolean denoting
-    whether the corresponding argument has a default value. For
+    by the function *f*. The value of each attribute is a Boolean
+    denoting whether the corresponding argument has a default value. For
     instance, `functionArgs ({ x, y ? 123}: ...) = { x = false; y =
     true; }`.
     
@@ -296,7 +296,7 @@ For instance, `derivation` is also available as `builtins.derivation`.
     the function. Plain lambdas are not included, e.g. `functionArgs (x:
     ...) = { }`.
 
-  - `builtins.fromJSON` e  
+  - `builtins.fromJSON` *e*  
     Convert a JSON string to a Nix value. For example,
     
         builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
@@ -304,21 +304,22 @@ For instance, `derivation` is also available as `builtins.derivation`.
     returns the value `{ x = [ 1 2 3 ]; y = null;
                     }`.
 
-  - `builtins.genList` generator length  
-    Generate list of size length, with each element i equal to the value
-    returned by generator `i`. For example,
+  - `builtins.genList` *generator* *length*  
+    Generate list of size *length*, with each element *i* equal to the
+    value returned by *generator* `i`. For example,
     
         builtins.genList (x: x * x) 5
     
     returns the list `[ 0 1 4 9 16 ]`.
 
-  - `builtins.getAttr` s set  
-    `getAttr` returns the attribute named s from set. Evaluation aborts
-    if the attribute doesn’t exist. This is a dynamic version of the `.`
-    operator, since s is an expression rather than an identifier.
+  - `builtins.getAttr` *s* *set*  
+    `getAttr` returns the attribute named *s* from *set*. Evaluation
+    aborts if the attribute doesn’t exist. This is a dynamic version of
+    the `.` operator, since *s* is an expression rather than an
+    identifier.
 
-  - `builtins.getEnv` s  
-    `getEnv` returns the value of the environment variable s, or an
+  - `builtins.getEnv` *s*  
+    `getEnv` returns the value of the environment variable *s*, or an
     empty string if the variable doesn’t exist. This function should be
     used with care, as it can introduce all sorts of nasty environment
     dependencies in your Nix expression.
@@ -328,29 +329,29 @@ For instance, `derivation` is also available as `builtins.derivation`.
     Packages. (That is, it does a `getEnv "HOME"` to locate the user’s
     home directory.)
 
-  - `builtins.hasAttr` s set  
-    `hasAttr` returns `true` if set has an attribute named s, and
+  - `builtins.hasAttr` *s* *set*  
+    `hasAttr` returns `true` if *set* has an attribute named *s*, and
     `false` otherwise. This is a dynamic version of the `?` operator,
-    since s is an expression rather than an identifier.
+    since *s* is an expression rather than an identifier.
 
-  - `builtins.hashString` type s  
+  - `builtins.hashString` *type* *s*  
     Return a base-16 representation of the cryptographic hash of string
-    s. The hash algorithm specified by type must be one of `"md5"`,
+    *s*. The hash algorithm specified by *type* must be one of `"md5"`,
     `"sha1"`, `"sha256"` or `"sha512"`.
 
-  - `builtins.hashFile` type p  
+  - `builtins.hashFile` *type* *p*  
     Return a base-16 representation of the cryptographic hash of the
-    file at path p. The hash algorithm specified by type must be one of
-    `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`.
+    file at path *p*. The hash algorithm specified by *type* must be one
+    of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`.
 
-  - `builtins.head` list  
+  - `builtins.head` *list*  
     Return the first element of a list; abort evaluation if the argument
     isn’t a list or is an empty list. You can test whether a list is
     empty by comparing it with `[]`.
 
-  - `import` path; `builtins.import` path  
-    Load, parse and return the Nix expression in the file path. If path
-    is a directory, the file ` default.nix
+  - `import` *path*; `builtins.import` *path*  
+    Load, parse and return the Nix expression in the file *path*. If
+    *path* is a directory, the file ` default.nix
                      ` in that directory is loaded. Evaluation aborts if the file
     doesn’t exist or contains an incorrect Nix expression. `import`
     implements Nix’s module system: you can put any Nix expression (such
@@ -361,7 +362,8 @@ For instance, `derivation` is also available as `builtins.derivation`.
     > 
     > Unlike some languages, `import` is a regular function in Nix.
     > Paths using the angle bracket syntax (e.g., `
-    >     >     >     >     > import` \<foo\>) are normal path values (see [???](#ssec-values)).
+    >     >     >     >     > import` *\<foo\>*) are normal path values (see
+    > [???](#ssec-values)).
     
     A Nix expression loaded by `import` must not contain any *free
     variables* (identifiers that are not defined in the Nix expression
@@ -393,50 +395,50 @@ For instance, `derivation` is also available as `builtins.derivation`.
     (The function argument doesn’t have to be called `x` in `foo.nix`;
     any name would work.)
 
-  - `builtins.intersectAttrs` e1 e2  
-    Return a set consisting of the attributes in the set e2 that also
-    exist in the set e1.
+  - `builtins.intersectAttrs` *e1* *e2*  
+    Return a set consisting of the attributes in the set *e2* that also
+    exist in the set *e1*.
 
-  - `builtins.isAttrs` e  
-    Return `true` if e evaluates to a set, and `false` otherwise.
+  - `builtins.isAttrs` *e*  
+    Return `true` if *e* evaluates to a set, and `false` otherwise.
 
-  - `builtins.isList` e  
-    Return `true` if e evaluates to a list, and `false` otherwise.
+  - `builtins.isList` *e*  
+    Return `true` if *e* evaluates to a list, and `false` otherwise.
 
-  - `builtins.isFunction` e  
-    Return `true` if e evaluates to a function, and `false` otherwise.
+  - `builtins.isFunction` *e*  
+    Return `true` if *e* evaluates to a function, and `false` otherwise.
 
-  - `builtins.isString` e  
-    Return `true` if e evaluates to a string, and `false` otherwise.
+  - `builtins.isString` *e*  
+    Return `true` if *e* evaluates to a string, and `false` otherwise.
 
-  - `builtins.isInt` e  
-    Return `true` if e evaluates to an int, and `false` otherwise.
+  - `builtins.isInt` *e*  
+    Return `true` if *e* evaluates to an int, and `false` otherwise.
 
-  - `builtins.isFloat` e  
-    Return `true` if e evaluates to a float, and `false` otherwise.
+  - `builtins.isFloat` *e*  
+    Return `true` if *e* evaluates to a float, and `false` otherwise.
 
-  - `builtins.isBool` e  
-    Return `true` if e evaluates to a bool, and `false` otherwise.
+  - `builtins.isBool` *e*  
+    Return `true` if *e* evaluates to a bool, and `false` otherwise.
 
-  - `builtins.isPath` e  
-    Return `true` if e evaluates to a path, and `false` otherwise.
+  - `builtins.isPath` *e*  
+    Return `true` if *e* evaluates to a path, and `false` otherwise.
 
-  - `isNull` e; `builtins.isNull` e  
-    Return `true` if e evaluates to `null`, and `false` otherwise.
+  - `isNull` *e*; `builtins.isNull` *e*  
+    Return `true` if *e* evaluates to `null`, and `false` otherwise.
     
     > **Warning**
     > 
     > This function is *deprecated*; just write `e == null` instead.
 
-  - `builtins.length` e  
-    Return the length of the list e.
+  - `builtins.length` *e*  
+    Return the length of the list *e*.
 
-  - `builtins.lessThan` e1 e2  
-    Return `true` if the number e1 is less than the number e2, and
-    `false` otherwise. Evaluation aborts if either e1 or e2 does not
+  - `builtins.lessThan` *e1* *e2*  
+    Return `true` if the number *e1* is less than the number *e2*, and
+    `false` otherwise. Evaluation aborts if either *e1* or *e2* does not
     evaluate to a number.
 
-  - `builtins.listToAttrs` e  
+  - `builtins.listToAttrs` *e*  
     Construct a set from a list specifying the names and values of each
     attribute. Each element of the list should be a set consisting of a
     string-valued attribute `name` specifying the name of the attribute,
@@ -451,19 +453,20 @@ For instance, `derivation` is also available as `builtins.derivation`.
     
         { foo = 123; bar = 456; }
 
-  - `map` f list; `builtins.map` f list  
-    Apply the function f to each element in the list list. For example,
+  - `map` *f* *list*; `builtins.map` *f* *list*  
+    Apply the function *f* to each element in the list *list*. For
+    example,
     
         map (x: "foo" + x) [ "bar" "bla" "abc" ]
     
     evaluates to `[ "foobar" "foobla" "fooabc"
                     ]`.
 
-  - `builtins.match` regex str  
+  - `builtins.match` *regex* *str*  
     Returns a list if the [extended POSIX regular
     expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
-    regex matches str precisely, otherwise returns `null`. Each item in
-    the list is a regex group.
+    *regex* matches *str* precisely, otherwise returns `null`. Each item
+    in the list is a regex group.
     
         builtins.match "ab" "abc"
     
@@ -481,21 +484,21 @@ For instance, `derivation` is also available as `builtins.derivation`.
     
     Evaluates to `[ "foo" ]`.
 
-  - `builtins.mul` e1 e2  
-    Return the product of the numbers e1 and e2.
+  - `builtins.mul` *e1* *e2*  
+    Return the product of the numbers *e1* and *e2*.
 
-  - `builtins.parseDrvName` s  
-    Split the string s into a package name and version. The package name
-    is everything up to but not including the first dash followed by a
-    digit, and the version is everything following that dash. The result
-    is returned in a set `{ name, version }`. Thus,
+  - `builtins.parseDrvName` *s*  
+    Split the string *s* into a package name and version. The package
+    name is everything up to but not including the first dash followed
+    by a digit, and the version is everything following that dash. The
+    result is returned in a set `{ name, version }`. Thus,
     `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = "nix";
     version = "0.12pre12876";
                     }`.
 
-  - `builtins.path` args  
+  - `builtins.path` *args*  
     An enrichment of the built-in path type, based on the attributes
-    present in args. All are optional except `path`:
+    present in *args*. All are optional except `path`:
     
       - path  
         The underlying path.
@@ -523,20 +526,20 @@ For instance, `derivation` is also available as `builtins.derivation`.
         providing a hash allows `builtins.path` to be used even when the
         `pure-eval` nix config option is on.
 
-  - `builtins.pathExists` path  
-    Return `true` if the path path exists at evaluation time, and
+  - `builtins.pathExists` *path*  
+    Return `true` if the path *path* exists at evaluation time, and
     `false` otherwise.
 
-  - `builtins.placeholder` output  
-    Return a placeholder string for the specified output that will be
+  - `builtins.placeholder` *output*  
+    Return a placeholder string for the specified *output* that will be
     substituted by the corresponding output path at build time. Typical
     outputs would be `"out"`, `"bin"` or `"dev"`.
 
-  - `builtins.readDir` path  
-    Return the contents of the directory path as a set mapping directory
-    entries to the corresponding file type. For instance, if directory
-    `A` contains a regular file `B` and another directory `C`, then
-    `builtins.readDir
+  - `builtins.readDir` *path*  
+    Return the contents of the directory *path* as a set mapping
+    directory entries to the corresponding file type. For instance, if
+    directory `A` contains a regular file `B` and another directory `C`,
+    then `builtins.readDir
                     ./A` will return the set
     
         { B = "regular"; C = "directory"; }
@@ -544,33 +547,33 @@ For instance, `derivation` is also available as `builtins.derivation`.
     The possible values for the file type are `"regular"`,
     `"directory"`, `"symlink"` and `"unknown"`.
 
-  - `builtins.readFile` path  
-    Return the contents of the file path as a string.
+  - `builtins.readFile` *path*  
+    Return the contents of the file *path* as a string.
 
-  - `removeAttrs` set list; `builtins.removeAttrs` set list  
-    Remove the attributes listed in list from set. The attributes don’t
-    have to exist in set. For instance,
+  - `removeAttrs` *set* *list*; `builtins.removeAttrs` *set* *list*  
+    Remove the attributes listed in *list* from *set*. The attributes
+    don’t have to exist in *set*. For instance,
     
         removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]
     
     evaluates to `{ y = 2; }`.
 
-  - `builtins.replaceStrings` from to s  
-    Given string s, replace every occurrence of the strings in from with
-    the corresponding string in to. For example,
+  - `builtins.replaceStrings` *from* *to* *s*  
+    Given string *s*, replace every occurrence of the strings in *from*
+    with the corresponding string in *to*. For example,
     
         builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
     
     evaluates to `"fabir"`.
 
-  - `builtins.seq` e1 e2  
-    Evaluate e1, then evaluate and return e2. This ensures that a
-    computation is strict in the value of e1.
+  - `builtins.seq` *e1* *e2*  
+    Evaluate *e1*, then evaluate and return *e2*. This ensures that a
+    computation is strict in the value of *e1*.
 
-  - `builtins.sort` comparator list  
-    Return list in sorted order. It repeatedly calls the function
-    comparator with two elements. The comparator should return `true` if
-    the first element is less than the second, and `false` otherwise.
+  - `builtins.sort` *comparator* *list*  
+    Return *list* in sorted order. It repeatedly calls the function
+    *comparator* with two elements. The comparator should return `true`
+    if the first element is less than the second, and `false` otherwise.
     For example,
     
         builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
@@ -581,12 +584,12 @@ For instance, `derivation` is also available as `builtins.derivation`.
     This is a stable sort: it preserves the relative order of elements
     deemed equal by the comparator.
 
-  - `builtins.split` regex str  
+  - `builtins.split` *regex* *str*  
     Returns a list composed of non matched strings interleaved with the
     lists of the [extended POSIX regular
     expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
-    regex matches of str. Each item in the lists of matched sequences is
-    a regex group.
+    *regex* matches of *str*. Each item in the lists of matched
+    sequences is a regex group.
     
         builtins.split "(a)b" "abc"
     
@@ -604,44 +607,44 @@ For instance, `derivation` is also available as `builtins.derivation`.
     
     Evaluates to `[ " " [ "FOO" ] " " ]`.
 
-  - `builtins.splitVersion` s  
+  - `builtins.splitVersion` *s*  
     Split a string representing a version into its components, by the
     same version splitting logic underlying the version comparison in
     [`nix-env -u`](#ssec-version-comparisons).
 
-  - `builtins.stringLength` e  
-    Return the length of the string e. If e is not a string, evaluation
-    is aborted.
+  - `builtins.stringLength` *e*  
+    Return the length of the string *e*. If *e* is not a string,
+    evaluation is aborted.
 
-  - `builtins.sub` e1 e2  
-    Return the difference between the numbers e1 and e2.
+  - `builtins.sub` *e1* *e2*  
+    Return the difference between the numbers *e1* and *e2*.
 
-  - `builtins.substring` start len s  
-    Return the substring of s from character position start (zero-based)
-    up to but not including start + len. If start is greater than the
-    length of the string, an empty string is returned, and if start +
-    len lies beyond the end of the string, only the substring up to the
-    end of the string is returned. start must be non-negative. For
-    example,
+  - `builtins.substring` *start* *len* *s*  
+    Return the substring of *s* from character position *start*
+    (zero-based) up to but not including *start + len*. If *start* is
+    greater than the length of the string, an empty string is returned,
+    and if *start + len* lies beyond the end of the string, only the
+    substring up to the end of the string is returned. *start* must be
+    non-negative. For example,
     
         builtins.substring 0 3 "nixos"
     
     evaluates to `"nix"`.
 
-  - `builtins.tail` list  
+  - `builtins.tail` *list*  
     Return the second to last elements of a list; abort evaluation if
     the argument isn’t a list or is an empty list.
 
-  - `throw` s; `builtins.throw` s  
-    Throw an error message s. This usually aborts Nix expression
+  - `throw` *s*; `builtins.throw` *s*  
+    Throw an error message *s*. This usually aborts Nix expression
     evaluation, but in `nix-env -qa` and other commands that try to
     evaluate a set of derivations to get information about those
     derivations, a derivation that throws an error is silently skipped
     (which is not the case for `abort`).
 
-  - `builtins.toFile` name s  
-    Store the string s in a file in the Nix store and return its path.
-    The file has suffix name. This file can be used as an input to
+  - `builtins.toFile` *name* *s*  
+    Store the string *s* in a file in the Nix store and return its path.
+    The file has suffix *name*. This file can be used as an input to
     derivations. One application is to write builders “inline”. For
     instance, the following Nix expression combines [???](#ex-hello-nix)
     and [???](#ex-hello-builder) into one file:
@@ -705,20 +708,20 @@ For instance, `derivation` is also available as `builtins.derivation`.
     you are using Nixpkgs, the `writeTextFile` function is able to do
     that.
 
-  - `builtins.toJSON` e  
-    Return a string containing a JSON representation of e. Strings,
+  - `builtins.toJSON` *e*  
+    Return a string containing a JSON representation of *e*. Strings,
     integers, floats, booleans, nulls and lists are mapped to their JSON
     equivalents. Sets (except derivations) are represented as objects.
     Derivations are translated to a JSON string containing the
     derivation’s output path. Paths are copied to the store and
     represented as a JSON string of the resulting store path.
 
-  - `builtins.toPath` s  
+  - `builtins.toPath` *s*  
     DEPRECATED. Use `/. + "/path"` to convert a string into an absolute
     path. For relative paths, use `./. + "/path"`.
 
-  - `toString` e; `builtins.toString` e  
-    Convert the expression e to a string. e can be:
+  - `toString` *e*; `builtins.toString` *e*  
+    Convert the expression *e* to a string. *e* can be:
     
       - A string (in which case the string is returned unmodified).
     
@@ -735,8 +738,8 @@ For instance, `derivation` is also available as `builtins.derivation`.
     
       - `null`, which yields the empty string.
 
-  - `builtins.toXML` e  
-    Return a string containing an XML representation of e. The main
+  - `builtins.toXML` *e*  
+    Return a string containing an XML representation of *e*. The main
     application for `toXML` is to communicate information with the
     builder in a more structured format than plain environment
     variables.
@@ -822,22 +825,23 @@ For instance, `derivation` is also available as `builtins.derivation`.
     of the stylesheet is spliced into the builder using the syntax
     `xsltproc ${stylesheet}`.
 
-  - `builtins.trace` e1 e2  
-    Evaluate e1 and print its abstract syntax representation on standard
-    error. Then return e2. This function is useful for debugging.
+  - `builtins.trace` *e1* *e2*  
+    Evaluate *e1* and print its abstract syntax representation on
+    standard error. Then return *e2*. This function is useful for
+    debugging.
 
-  - `builtins.tryEval` e  
-    Try to shallowly evaluate e. Return a set containing the attributes
-    `success` (`true` if e evaluated successfully, `false` if an error
-    was thrown) and `value`, equalling e if successful and `false`
-    otherwise. Note that this doesn't evaluate e deeply, so ` let e = {
-    x = throw ""; }; in (builtins.tryEval e).success
+  - `builtins.tryEval` *e*  
+    Try to shallowly evaluate *e*. Return a set containing the
+    attributes `success` (`true` if *e* evaluated successfully, `false`
+    if an error was thrown) and `value`, equalling *e* if successful and
+    `false` otherwise. Note that this doesn't evaluate *e* deeply, so
+    ` let e = { x = throw ""; }; in (builtins.tryEval e).success
                      ` will be `true`. Using ` builtins.deepSeq
                      ` one can get the expected result: `let e = { x = throw "";
                     }; in (builtins.tryEval (builtins.deepSeq e e)).success` will be
     `false`.
 
-  - `builtins.typeOf` e  
-    Return a string representing the type of the value e, namely
+  - `builtins.typeOf` *e*  
+    Return a string representing the type of the value *e*, namely
     `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`,
     `"list"`, `"lambda"` or `"float"`.
diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md
index 9e99a1f60..2abe6ac6e 100644
--- a/doc/manual/src/expressions/expression-syntax.md
+++ b/doc/manual/src/expressions/expression-syntax.md
@@ -33,7 +33,7 @@ elements (referenced from the figure by number):
     
     Nix functions generally have the form `{ x, y, ...,
                     z }: e` where `x`, `y`, etc. are the names of the expected
-    arguments, and where e is the body of the function. So here, the
+    arguments, and where *e* is the body of the function. So here, the
     entire remainder of the file is the body of the function; when given
     the required arguments, the body should describe how to build an
     instance of the Hello package.
diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/expressions/language-constructs.md
index 20d003348..2699d675f 100644
--- a/doc/manual/src/expressions/language-constructs.md
+++ b/doc/manual/src/expressions/language-constructs.md
@@ -147,7 +147,7 @@ three kinds of patterns:
     It is possible to provide *default values* for attributes, in which
     case they are allowed to be missing. A default value is specified by
     writing `name ?
-            e`, where e is an arbitrary expression. For example,
+            e`, where *e* is an arbitrary expression. For example,
     
         { x, y ? "foo", z ? "bar" }: z + y + x
     
@@ -201,7 +201,7 @@ Conditionals look like this:
 
     if e1 then e2 else e3
 
-where e1 is an expression that should evaluate to a Boolean value
+where *e1* is an expression that should evaluate to a Boolean value
 (`true` or `false`).
 
 ## Assertions
@@ -211,9 +211,9 @@ between features and dependencies hold. They look like this:
 
     assert e1; e2
 
-where e1 is an expression that should evaluate to a Boolean value. If it
-evaluates to `true`, e2 is returned; otherwise expression evaluation is
-aborted and a backtrace is printed.
+where *e1* is an expression that should evaluate to a Boolean value. If
+it evaluates to `true`, *e2* is returned; otherwise expression
+evaluation is aborted and a backtrace is printed.
 
 Here is a Nix expression for the Subversion package that shows how
 assertions can be used:.
@@ -275,8 +275,8 @@ A *with-expression*,
 
     with e1; e2
 
-introduces the set e1 into the lexical scope of the expression e2. For
-instance,
+introduces the set *e1* into the lexical scope of the expression *e2*.
+For instance,
 
     let as = { x = "foo"; y = "bar"; };
     in with as; x + y
diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md
index 4fa2eca37..b124a2417 100644
--- a/doc/manual/src/expressions/language-operators.md
+++ b/doc/manual/src/expressions/language-operators.md
@@ -4,29 +4,29 @@
 expression language, in order of precedence (from strongest to weakest
 binding).
 
-| Name                     | Syntax                        | Associativity | Description                                                                                                                                                                                                             | Precedence |
-| ------------------------ | ----------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
-| Select                   | e `.` attrpath \[ `or` def \] | none          | Select attribute denoted by the attribute path attrpath from set e. (An attribute path is a dot-separated list of attribute names.) If the attribute doesn’t exist, return def if provided, otherwise abort evaluation. | 1          |
-| Application              | e1 e2                         | left          | Call function e1 with argument e2.                                                                                                                                                                                      | 2          |
-| Arithmetic Negation      | `-` e                         | none          | Arithmetic negation.                                                                                                                                                                                                    | 3          |
-| Has Attribute            | e `?` attrpath                | none          | Test whether set e contains the attribute denoted by attrpath; return `true` or `false`.                                                                                                                                | 4          |
-| List Concatenation       | e1 `++` e2                    | right         | List concatenation.                                                                                                                                                                                                     | 5          |
-| Multiplication           | e1 `*` e2,                    | left          | Arithmetic multiplication.                                                                                                                                                                                              | 6          |
-| Division                 | e1 `/` e2                     | left          | Arithmetic division.                                                                                                                                                                                                    | 6          |
-| Addition                 | e1 `+` e2                     | left          | Arithmetic addition.                                                                                                                                                                                                    | 7          |
-| Subtraction              | e1 `-` e2                     | left          | Arithmetic subtraction.                                                                                                                                                                                                 | 7          |
-| String Concatenation     | string1 `+` string2           | left          | String concatenation.                                                                                                                                                                                                   | 7          |
-| Not                      | `!` e                         | none          | Boolean negation.                                                                                                                                                                                                       | 8          |
-| Update                   | e1 `//` e2                    | right         | Return a set consisting of the attributes in e1 and e2 (with the latter taking precedence over the former in case of equally named attributes).                                                                         | 9          |
-| Less Than                | e1 `<` e2,                    | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
-| Less Than or Equal To    | e1 `<=` e2                    | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
-| Greater Than             | e1 `>` e2                     | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
-| Greater Than or Equal To | e1 `>=` e2                    | none          | Arithmetic comparison.                                                                                                                                                                                                  | 10         |
-| Equality                 | e1 `==` e2                    | none          | Equality.                                                                                                                                                                                                               | 11         |
-| Inequality               | e1 `!=` e2                    | none          | Inequality.                                                                                                                                                                                                             | 11         |
-| Logical AND              | e1 `&&` e2                    | left          | Logical AND.                                                                                                                                                                                                            | 12         |
-| Logical OR               | e1 `\|\|` e2                  | left          | Logical OR.                                                                                                                                                                                                             | 13         |
-| Logical Implication      | e1 `->` e2                    | none          | Logical implication (equivalent to `!e1 \|\|
-        e2`).                                                                                                                                                              | 14         |
+| Name                     | Syntax                              | Associativity | Description                                                                                                                                                                                                                   | Precedence |
+| ------------------------ | ----------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
+| Select                   | *e* `.` *attrpath* \[ `or` *def* \] | none          | Select attribute denoted by the attribute path *attrpath* from set *e*. (An attribute path is a dot-separated list of attribute names.) If the attribute doesn’t exist, return *def* if provided, otherwise abort evaluation. | 1          |
+| Application              | *e1* *e2*                           | left          | Call function *e1* with argument *e2*.                                                                                                                                                                                        | 2          |
+| Arithmetic Negation      | `-` *e*                             | none          | Arithmetic negation.                                                                                                                                                                                                          | 3          |
+| Has Attribute            | *e* `?` *attrpath*                  | none          | Test whether set *e* contains the attribute denoted by *attrpath*; return `true` or `false`.                                                                                                                                  | 4          |
+| List Concatenation       | *e1* `++` *e2*                      | right         | List concatenation.                                                                                                                                                                                                           | 5          |
+| Multiplication           | *e1* `*` *e2*,                      | left          | Arithmetic multiplication.                                                                                                                                                                                                    | 6          |
+| Division                 | *e1* `/` *e2*                       | left          | Arithmetic division.                                                                                                                                                                                                          | 6          |
+| Addition                 | *e1* `+` *e2*                       | left          | Arithmetic addition.                                                                                                                                                                                                          | 7          |
+| Subtraction              | *e1* `-` *e2*                       | left          | Arithmetic subtraction.                                                                                                                                                                                                       | 7          |
+| String Concatenation     | *string1* `+` *string2*             | left          | String concatenation.                                                                                                                                                                                                         | 7          |
+| Not                      | `!` *e*                             | none          | Boolean negation.                                                                                                                                                                                                             | 8          |
+| Update                   | *e1* `//` *e2*                      | right         | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes).                                                                           | 9          |
+| Less Than                | *e1* `<` *e2*,                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
+| Less Than or Equal To    | *e1* `<=` *e2*                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
+| Greater Than             | *e1* `>` *e2*                       | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
+| Greater Than or Equal To | *e1* `>=` *e2*                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
+| Equality                 | *e1* `==` *e2*                      | none          | Equality.                                                                                                                                                                                                                     | 11         |
+| Inequality               | *e1* `!=` *e2*                      | none          | Inequality.                                                                                                                                                                                                                   | 11         |
+| Logical AND              | *e1* `&&` *e2*                      | left          | Logical AND.                                                                                                                                                                                                                  | 12         |
+| Logical OR               | *e1* `\|\|` *e2*                    | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
+| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to `!e1 \|\|
+        e2`).                                                                                                                                                                    | 14         |
 
 Operators
diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md
index c498713de..35dbd5541 100644
--- a/doc/manual/src/installation/building-source.md
+++ b/doc/manual/src/installation/building-source.md
@@ -17,7 +17,7 @@ command:
 The installation path can be specified by passing the `--prefix=prefix`
 to `configure`. The default installation directory is `/usr/local`. You
 can change this to any location you like. You must have write permission
-to the prefix path.
+to the *prefix* path.
 
 Nix keeps its *store* (the place where packages are stored) in
 `/nix/store` by default. This can be changed using
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
index 04dad3339..d9c34afc6 100644
--- a/doc/manual/src/package-management/basic-package-mgmt.md
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -60,7 +60,8 @@ Nixpkgs tree using the `-f` flag:
 
     $ nix-env -qaf /path/to/nixpkgs
 
-where /path/to/nixpkgs is where you’ve unpacked or checked out Nixpkgs.
+where */path/to/nixpkgs* is where you’ve unpacked or checked out
+Nixpkgs.
 
 You can select specific packages by name:
 
diff --git a/doc/manual/src/release-notes/rl-0.10.md b/doc/manual/src/release-notes/rl-0.10.md
index 07b19ff05..1301add26 100644
--- a/doc/manual/src/release-notes/rl-0.10.md
+++ b/doc/manual/src/release-notes/rl-0.10.md
@@ -34,7 +34,7 @@
     
       - `nix-env -i
                                                         pkgname` will now install the highest available version of
-        pkgname, rather than installing all available versions (which
+        *pkgname*, rather than installing all available versions (which
         would probably give collisions) (`NIX-31`).
     
       - `nix-env (-i|-u) --dry-run` now shows exactly which missing
diff --git a/doc/manual/src/release-notes/rl-0.12.md b/doc/manual/src/release-notes/rl-0.12.md
index aaecfbb70..3a4aba07d 100644
--- a/doc/manual/src/release-notes/rl-0.12.md
+++ b/doc/manual/src/release-notes/rl-0.12.md
@@ -41,10 +41,10 @@
 
   - The garbage collector has a number of new options to allow only some
     of the garbage to be deleted. The option `--max-freed N` tells the
-    collector to stop after at least N bytes have been deleted. The
+    collector to stop after at least *N* bytes have been deleted. The
     option `--max-links
             N` tells it to stop after the link count on `/nix/store` has dropped
-    below N. This is useful for very large Nix stores on filesystems
+    below *N*. This is useful for very large Nix stores on filesystems
     with a 32000 subdirectories limit (like `ext3`). The option
     `--use-atime` causes store paths to be deleted in order of ascending
     last access time. This allows non-recently used stuff to be deleted.
diff --git a/doc/manual/src/release-notes/rl-0.13.md b/doc/manual/src/release-notes/rl-0.13.md
index 21f33e3db..13a60e01c 100644
--- a/doc/manual/src/release-notes/rl-0.13.md
+++ b/doc/manual/src/release-notes/rl-0.13.md
@@ -51,5 +51,5 @@ This is primarily a bug fix release. It has some new features:
     option is used.
 
   - The scoping rules for `inherit
-                    (e) ...` in recursive attribute sets have changed. The expression e
-    can now refer to the attributes defined in the containing set.
+                    (e) ...` in recursive attribute sets have changed. The expression
+    *e* can now refer to the attributes defined in the containing set.
diff --git a/doc/manual/src/release-notes/rl-0.16.md b/doc/manual/src/release-notes/rl-0.16.md
index 79074fcdc..23ac53786 100644
--- a/doc/manual/src/release-notes/rl-0.16.md
+++ b/doc/manual/src/release-notes/rl-0.16.md
@@ -15,7 +15,7 @@ This release has the following improvements:
     `--cores
                             N` as well as a configuration setting `build-cores =
                             N` that causes the environment variable `NIX_BUILD_CORES` to be set
-    to N when the builder is invoked. The builder can use this at its
+    to *N* when the builder is invoked. The builder can use this at its
     discretion to perform a parallel build, e.g., by calling `make -j
                             N`. In Nixpkgs, this can be enabled on a per-package basis by
     setting the derivation attribute `enableParallelBuilding` to `true`.
diff --git a/doc/manual/src/release-notes/rl-0.6.md b/doc/manual/src/release-notes/rl-0.6.md
index c816c9851..ed2d21583 100644
--- a/doc/manual/src/release-notes/rl-0.6.md
+++ b/doc/manual/src/release-notes/rl-0.6.md
@@ -49,8 +49,8 @@
     
       - New language construct: `with
                                                         E1;
-                                                        E2` brings all attributes defined in the attribute set E1 in
-        scope in E2.
+                                                        E2` brings all attributes defined in the attribute set *E1* in
+        scope in *E2*.
     
       - Added a `map` function.
     
diff --git a/doc/manual/src/release-notes/rl-1.11.md b/doc/manual/src/release-notes/rl-1.11.md
index 381887bd8..fbabdaa2f 100644
--- a/doc/manual/src/release-notes/rl-1.11.md
+++ b/doc/manual/src/release-notes/rl-1.11.md
@@ -35,7 +35,7 @@ features:
     derivations, and in `builtins.hashString`.
 
   - The new flag `--option build-repeat
-                            N` will cause every build to be executed N+1 times. If the build
+                            N` will cause every build to be executed *N*+1 times. If the build
     output differs between any round, the build is rejected, and the
     output paths are not registered as valid. This is primarily useful
     to verify build determinism. (We already had a `--check` option to
diff --git a/doc/manual/src/release-notes/rl-1.4.md b/doc/manual/src/release-notes/rl-1.4.md
index d6d2227f8..d23de71ad 100644
--- a/doc/manual/src/release-notes/rl-1.4.md
+++ b/doc/manual/src/release-notes/rl-1.4.md
@@ -10,7 +10,7 @@ There are also the following improvements:
 
   - New built-in function: `builtins.hashString`.
 
-  - Build logs are now stored in `/nix/var/log/nix/drvs/XX/`, where XX
+  - Build logs are now stored in `/nix/var/log/nix/drvs/XX/`, where *XX*
     is the first two characters of the derivation. This is useful on
     machines that keep a lot of build logs (such as Hydra servers).
 
diff --git a/doc/manual/src/release-notes/rl-1.6.1.md b/doc/manual/src/release-notes/rl-1.6.1.md
index ed974fe0b..9bb9bb1f8 100644
--- a/doc/manual/src/release-notes/rl-1.6.1.md
+++ b/doc/manual/src/release-notes/rl-1.6.1.md
@@ -6,8 +6,8 @@ This is primarily a bug fix release. Changes of interest are:
     strings, such as `"${/foo}/bar"`. This release reverts to the Nix
     1.5.3 behaviour.
 
-  - Previously, Nix optimised expressions such as `"${expr}"` to expr.
-    Thus it neither checked whether expr could be coerced to a string,
+  - Previously, Nix optimised expressions such as `"${expr}"` to *expr*.
+    Thus it neither checked whether *expr* could be coerced to a string,
     nor applied such coercions. This meant that `"${123}"` evaluatued to
     `123`, and `"${./foo}"` evaluated to `./foo` (even though `"${./foo}
     "` evaluates to `"/nix/store/hash-foo "`). Nix now checks the type
diff --git a/doc/manual/src/release-notes/rl-1.7.md b/doc/manual/src/release-notes/rl-1.7.md
index 71a327ed6..8d49ae54e 100644
--- a/doc/manual/src/release-notes/rl-1.7.md
+++ b/doc/manual/src/release-notes/rl-1.7.md
@@ -89,10 +89,10 @@ features:
     specifier. For example, `nix-store --gc --max-freed
                     1G` will free up to 1 gigabyte of disk space.
 
-  - `nix-collect-garbage` has a new flag `--delete-older-than` N`d`,
-    which deletes all user environment generations older than N days.
+  - `nix-collect-garbage` has a new flag `--delete-older-than` *N*`d`,
+    which deletes all user environment generations older than *N* days.
     Likewise, `nix-env
-                    --delete-generations` accepts a N`d` age limit.
+                    --delete-generations` accepts a *N*`d` age limit.
 
   - Nix now heuristically detects whether a build failure was due to a
     disk-full condition. In that case, the build is not flagged as

From 69333cb62c0631919ad447629bdd1035df94eb67 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 14:30:13 +0200
Subject: [PATCH 035/882] Sigh

---
 doc/manual/src/SUMMARY.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 80a41f8d7..5f6817674 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -22,9 +22,9 @@
   - [Channels](package-management/channels.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
-    - [Copying Closures Via SSH](package-management/copy-closure.md)
+    - [Copying Closures via SSH](package-management/copy-closure.md)
     - [Serving a Nix store via SSH](package-management/ssh-substituter.md)
-    - [Serving a Nix store via AWS S3 or S3-compatible Service](package-management/s3-substituter.md)
+    - [Serving a Nix store via S3](package-management/s3-substituter.md)
 - [Writing Nix Expressions](expressions/writing-nix-expressions.md)
   - [A Simple Nix Expression](expressions/simple-expression.md)
     - [Expression Syntax](expressions/expression-syntax.md)

From 5e3ad1dde0a03b3bd094e1d4ecc0f4fc7abdaa5c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Jul 2020 14:55:10 +0200
Subject: [PATCH 036/882] Add a separate manual job

---
 flake.nix | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index cebbdf9d9..f31dc4039 100644
--- a/flake.nix
+++ b/flake.nix
@@ -144,11 +144,6 @@
 
           installFlags = "sysconfdir=$(out)/etc";
 
-          postInstall = ''
-            mkdir -p $doc/nix-support
-            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-          '';
-
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
@@ -215,6 +210,25 @@
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
 
+        # Separate build for just the manual.
+        manual =
+          with nixpkgsFor.x86_64-linux;
+          stdenv.mkDerivation {
+            name = "nix-manual-${version}";
+            src = self;
+            buildInputs = [ mdbook ];
+            configurePhase = ":";
+            buildPhase = ":";
+            installPhase =
+              ''
+                touch Makefile.config
+                docdir=$out/share/doc/nix
+                make docdir=$docdir doc_generate=yes $docdir/manual/index.html
+                mkdir -p $out/nix-support
+                echo "doc manual $docdir/manual" >> $out/nix-support/hydra-build-products
+              '';
+          };
+
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.

From e1de1fe0d82d8ba702947dcad3b678cbb9ce9333 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 23 Jul 2020 19:02:57 +0000
Subject: [PATCH 037/882] Make `Buildable` a `std::variant`

I think this better captures the intent of what's going on: we either
have an opaque store path, or a drv path with some outputs.

Having this structure will also help us support CA derivations: we'll
have to allow the outpath paths to be optional, so the structure we gain
now makes up for the structure we loose then.
---
 src/libstore/content-address.cc |  4 --
 src/libstore/store-api.cc       |  4 --
 src/libutil/util.hh             |  5 ++
 src/nix/build.cc                | 29 +++++++----
 src/nix/command.cc              | 17 ++++--
 src/nix/installables.cc         | 91 ++++++++++++++++-----------------
 src/nix/installables.hh         | 14 +++--
 src/nix/log.cc                  | 13 +++--
 8 files changed, 98 insertions(+), 79 deletions(-)

diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 6cb69d0a9..f45f77d5c 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -24,10 +24,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
         + hash.to_string(Base32, true);
 }
 
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
 std::string renderContentAddress(ContentAddress ca) {
     return std::visit(overloaded {
         [](TextHash th) {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ab21b0bd5..0f6f3b98f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -865,10 +865,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
     sigs.insert(secretKey.signDetached(fingerprint(store)));
 }
 
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
 bool ValidPathInfo::isContentAddressed(const Store & store) const
 {
     if (! ca) return false;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 42130f6dc..630303a5d 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -601,4 +601,9 @@ constexpr auto enumerate(T && iterable)
 }
 
 
+// C++17 std::visit boilerplate
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+
 }
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 0f7e0e123..927301314 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -57,17 +57,24 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 
         if (dryRun) return;
 
-        if (outLink != "") {
-            for (size_t i = 0; i < buildables.size(); ++i) {
-                for (auto & output : buildables[i].outputs)
-                    if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
-                        std::string symlink = outLink;
-                        if (i) symlink += fmt("-%d", i);
-                        if (output.first != "out") symlink += fmt("-%s", output.first);
-                        store2->addPermRoot(output.second, absPath(symlink), true);
-                    }
-            }
-        }
+        if (outLink != "")
+            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
+                for (size_t i = 0; i < buildables.size(); ++i)
+                    std::visit(overloaded {
+                        [&](BuildableOpaque bo) {
+                            std::string symlink = outLink;
+                            if (i) symlink += fmt("-%d", i);
+                            store2->addPermRoot(bo.path, absPath(symlink), true);
+                        },
+                        [&](BuildableFromDrv bfd) {
+                            for (auto & output : bfd.outputs) {
+                                std::string symlink = outLink;
+                                if (i) symlink += fmt("-%d", i);
+                                if (output.first != "out") symlink += fmt("-%s", output.first);
+                                store2->addPermRoot(output.second, absPath(symlink), true);
+                            }
+                        },
+                    }, buildables[i]);
 
         updateProfile(buildables);
     }
diff --git a/src/nix/command.cc b/src/nix/command.cc
index af36dda89..04715b43a 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -131,11 +131,18 @@ void MixProfile::updateProfile(const Buildables & buildables)
     std::optional<StorePath> result;
 
     for (auto & buildable : buildables) {
-        for (auto & output : buildable.outputs) {
-            if (result)
-                throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
-            result = output.second;
-        }
+        std::visit(overloaded {
+            [&](BuildableOpaque bo) {
+                result = bo.path;
+            },
+            [&](BuildableFromDrv bfd) {
+                for (auto & output : bfd.outputs) {
+                    if (result)
+                        throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
+                    result = output.second;
+                }
+            },
+        }, buildable);
     }
 
     if (!result)
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index a13e5a3df..a051950cd 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -307,16 +307,15 @@ struct InstallableStorePath : Installable
             for (auto & [name, output] : store->readDerivation(storePath).outputs)
                 outputs.emplace(name, output.path);
             return {
-                Buildable {
+                BuildableFromDrv {
                     .drvPath = storePath,
                     .outputs = std::move(outputs)
                 }
             };
         } else {
             return {
-                Buildable {
-                    .drvPath = {},
-                    .outputs = {{"out", storePath}}
+                BuildableOpaque {
+                    .path = storePath,
                 }
             };
         }
@@ -332,33 +331,20 @@ Buildables InstallableValue::toBuildables()
 {
     Buildables res;
 
-    StorePathSet drvPaths;
+    std::map<StorePath, OutputPathMap> drvsToOutputs;
 
+    // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
-        Buildable b{.drvPath = drv.drvPath};
-        drvPaths.insert(drv.drvPath);
-
         auto outputName = drv.outputName;
         if (outputName == "")
-            throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath));
-
-        b.outputs.emplace(outputName, drv.outPath);
-
-        res.push_back(std::move(b));
+            throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
+        drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
     }
 
-    // Hack to recognize .all: if all drvs have the same drvPath,
-    // merge the buildables.
-    if (drvPaths.size() == 1) {
-        Buildable b{.drvPath = *drvPaths.begin()};
-        for (auto & b2 : res)
-            for (auto & output : b2.outputs)
-                b.outputs.insert_or_assign(output.first, output.second);
-        Buildables bs;
-        bs.push_back(std::move(b));
-        return bs;
-    } else
-        return res;
+    for (auto & i : drvsToOutputs)
+        res.push_back(BuildableFromDrv { i.first, i.second });
+
+    return res;
 }
 
 struct InstallableAttrPath : InstallableValue
@@ -653,14 +639,17 @@ Buildables build(ref<Store> store, Realise mode,
 
     for (auto & i : installables) {
         for (auto & b : i->toBuildables()) {
-            if (b.drvPath) {
-                StringSet outputNames;
-                for (auto & output : b.outputs)
-                    outputNames.insert(output.first);
-                pathsToBuild.push_back({*b.drvPath, outputNames});
-            } else
-                for (auto & output : b.outputs)
-                    pathsToBuild.push_back({output.second});
+            std::visit(overloaded {
+                [&](BuildableOpaque bo) {
+                    pathsToBuild.push_back({bo.path});
+                },
+                [&](BuildableFromDrv bfd) {
+                    StringSet outputNames;
+                    for (auto & output : bfd.outputs)
+                        outputNames.insert(output.first);
+                    pathsToBuild.push_back({bfd.drvPath, outputNames});
+                },
+            }, b);
             buildables.push_back(std::move(b));
         }
     }
@@ -681,16 +670,23 @@ StorePathSet toStorePaths(ref<Store> store,
 
     if (operateOn == OperateOn::Output) {
         for (auto & b : build(store, mode, installables))
-            for (auto & output : b.outputs)
-                outPaths.insert(output.second);
+            std::visit(overloaded {
+                [&](BuildableOpaque bo) {
+                    outPaths.insert(bo.path);
+                },
+                [&](BuildableFromDrv bfd) {
+                    for (auto & output : bfd.outputs)
+                        outPaths.insert(output.second);
+                },
+            }, b);
     } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
         for (auto & i : installables)
             for (auto & b : i->toBuildables())
-                if (b.drvPath)
-                    outPaths.insert(*b.drvPath);
+                if (auto bfd = std::get_if<BuildableFromDrv>(&b))
+                    outPaths.insert(bfd->drvPath);
     }
 
     return outPaths;
@@ -714,20 +710,21 @@ StorePathSet toDerivations(ref<Store> store,
     StorePathSet drvPaths;
 
     for (auto & i : installables)
-        for (auto & b : i->toBuildables()) {
-            if (!b.drvPath) {
-                if (!useDeriver)
-                    throw Error("argument '%s' did not evaluate to a derivation", i->what());
-                for (auto & output : b.outputs) {
-                    auto derivers = store->queryValidDerivers(output.second);
+        for (auto & b : i->toBuildables())
+            std::visit(overloaded {
+                [&](BuildableOpaque bo) {
+                    if (!useDeriver)
+                        throw Error("argument '%s' did not evaluate to a derivation", i->what());
+                    auto derivers = store->queryValidDerivers(bo.path);
                     if (derivers.empty())
                         throw Error("'%s' does not have a known deriver", i->what());
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
-                }
-            } else
-                drvPaths.insert(*b.drvPath);
-        }
+                },
+                [&](BuildableFromDrv bfd) {
+                    drvPaths.insert(bfd.drvPath);
+                },
+            }, b);
 
     return drvPaths;
 }
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index eb34365d4..9edff3331 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -14,12 +14,20 @@ struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }
 
-struct Buildable
-{
-    std::optional<StorePath> drvPath;
+struct BuildableOpaque {
+    StorePath path;
+};
+
+struct BuildableFromDrv {
+    StorePath drvPath;
     std::map<std::string, StorePath> outputs;
 };
 
+typedef std::variant<
+    BuildableOpaque,
+    BuildableFromDrv
+> Buildable;
+
 typedef std::vector<Buildable> Buildables;
 
 struct App
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 7e10d373a..33380dcf5 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -45,11 +45,14 @@ struct CmdLog : InstallableCommand
 
         RunPager pager;
         for (auto & sub : subs) {
-            auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr;
-            for (auto & output : b.outputs) {
-                if (log) break;
-                log = sub->getBuildLog(output.second);
-            }
+            auto log = std::visit(overloaded {
+                [&](BuildableOpaque bo) {
+                    return sub->getBuildLog(bo.path);
+                },
+                [&](BuildableFromDrv bfd) {
+                    return sub->getBuildLog(bfd.drvPath);
+                },
+            }, b);
             if (!log) continue;
             stopProgressBar();
             printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());

From a71d1cedffe17ea4dd6a48c12cf5b9323ca6114c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 11:34:01 +0200
Subject: [PATCH 038/882] printVersion(): Show system types

---
 src/libmain/shared.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 52718c231..22929cae5 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -277,6 +277,8 @@ void printVersion(const string & programName)
 #if HAVE_SODIUM
         cfg.push_back("signed-caches");
 #endif
+        std::cout << "System type: " << settings.thisSystem << "\n";
+        std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
         std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
         std::cout << "User configuration files: " <<

From 8d0b311a1ccd0aef49c6f272aad4ecb5105b285a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 11:43:44 +0200
Subject: [PATCH 039/882] Get rid of footnotes

Markdown doesn't support them.
---
 doc/manual/command-ref/nix-instantiate.xml    |  2 +-
 doc/manual/command-ref/opt-common.xml         |  2 +-
 doc/manual/expressions/build-script.xml       | 11 ++++------
 doc/manual/expressions/derivations.xml        | 19 ++++++++---------
 doc/manual/expressions/generic-builder.xml    |  6 +++---
 .../expressions/language-constructs.xml       |  7 +++----
 doc/manual/expressions/language-values.xml    | 11 +++++-----
 doc/manual/expressions/simple-expression.xml  |  7 +++----
 doc/manual/packages/profiles.xml              | 13 ++++++------
 doc/manual/src/command-ref/nix-instantiate.md |  9 ++++----
 doc/manual/src/command-ref/opt-common.md      |  6 +++---
 doc/manual/src/expressions/build-script.md    | 10 +++------
 doc/manual/src/expressions/derivations.md     | 17 ++++++---------
 doc/manual/src/expressions/generic-builder.md | 15 ++++++-------
 .../src/expressions/language-constructs.md    |  7 ++-----
 doc/manual/src/expressions/language-values.md | 21 +++++++++----------
 .../src/expressions/simple-expression.md      | 10 +++------
 doc/manual/src/package-management/profiles.md | 16 +++++++-------
 18 files changed, 79 insertions(+), 110 deletions(-)

diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index d67be4924..dc055fa51 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -106,7 +106,7 @@ input.</para>
   <varlistentry><term><option>--find-file</option></term>
 
     <listitem><para>Look up the given files in Nix’s search path (as
-    specified by the <envar linkend="env-NIX_PATH">NIX_PATH</literal>
+    specified by the <literal linkend="env-NIX_PATH">NIX_PATH</literal>
     environment variable).  If found, print the corresponding absolute
     paths on standard output.  For instance, if
     <literal>NIX_PATH</literal> is
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 67950e94b..a095039b9 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -369,7 +369,7 @@
 <varlistentry xml:id="opt-I"><term><option>-I</option> <emphasis>path</emphasis></term>
 
   <listitem><para>Add a path to the Nix expression search path.  This
-  option may be given multiple times.  See the <envar
+  option may be given multiple times.  See the <literal
   linkend="env-NIX_PATH">NIX_PATH</literal> environment variable for
   information on the semantics of the Nix search path.  Paths added
   through <option>-I</option> take precedence over
diff --git a/doc/manual/expressions/build-script.xml b/doc/manual/expressions/build-script.xml
index c9af198f2..5c55954fc 100644
--- a/doc/manual/expressions/build-script.xml
+++ b/doc/manual/expressions/build-script.xml
@@ -33,13 +33,10 @@ steps:</para>
 
     <para>When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the
-    derivation).  For instance, the <literal>PATH</literal> variable is
-    empty<footnote><para>Actually, it's initialised to
-    <filename>/path-not-set</filename> to prevent Bash from setting it
-    to a default value.</para></footnote>.  This is done to prevent
-    undeclared inputs from being used in the build process.  If for
-    example the <literal>PATH</literal> contained
-    <filename>/usr/bin</filename>, then you might accidentally use
+    derivation).  This is done to prevent undeclared inputs from being
+    used in the build process. If for example the
+    <literal>PATH</literal> contained <filename>/usr/bin</filename>,
+    then you might accidentally use
     <filename>/usr/bin/gcc</filename>.</para>
 
     <para>So the first step is to set up the environment.  This is
diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml
index a11de0088..0625bcdfe 100644
--- a/doc/manual/expressions/derivations.xml
+++ b/doc/manual/expressions/derivations.xml
@@ -13,16 +13,15 @@ of which specify the inputs of the build.</para>
 
 <itemizedlist>
 
-  <listitem xml:id="attr-system"><para>There must be an attribute named
-  <varname>system</varname> whose value must be a string specifying a
-  Nix platform identifier, such as <literal>"i686-linux"</literal> or
-  <literal>"x86_64-darwin"</literal><footnote><para>To figure out
-  your platform identifier, look at the line <quote>Checking for the
-  canonical Nix system name</quote> in the output of Nix's
-  <filename>configure</filename> script.</para></footnote> The build
-  can only be performed on a machine and operating system matching the
-  platform identifier.  (Nix can automatically forward builds for
-  other platforms by forwarding them to other machines; see <xref
+  <listitem xml:id="attr-system"><para>There must be an attribute
+  named <varname>system</varname> whose value must be a string
+  specifying a Nix system type, such as
+  <literal>"i686-linux"</literal> or
+  <literal>"x86_64-darwin"</literal>. (To figure out your system type,
+  run <literal>nix -vv --version</literal>.) The build can only be
+  performed on a machine and operating system matching the system
+  type.  (Nix can automatically forward builds for other platforms by
+  forwarding them to other machines; see <xref
   linkend='chap-distributed-builds' />.)</para></listitem>
 
   <listitem><para>There must be an attribute named
diff --git a/doc/manual/expressions/generic-builder.xml b/doc/manual/expressions/generic-builder.xml
index f6e67813d..a6f258abc 100644
--- a/doc/manual/expressions/generic-builder.xml
+++ b/doc/manual/expressions/generic-builder.xml
@@ -42,14 +42,14 @@ genericBuild ③</programlisting>
     <filename>bin</filename> subdirectory, it's added to
     <literal>PATH</literal>; if it has a <filename>include</filename>
     subdirectory, it's added to GCC's header search path; and so
-    on.<footnote><para>How does it work? <filename>setup</filename>
-    tries to source the file
+    on. (This is implemented in a modular way:
+    <filename>setup</filename> tries to source the file
     <filename><emphasis>pkg</emphasis>/nix-support/setup-hook</filename>
     of all dependencies.  These “setup hooks” can then set up whatever
     environment variables they want; for instance, the setup hook for
     Perl sets the <literal>PERL5LIB</literal> environment variable to
     contain the <filename>lib/site_perl</filename> directories of all
-    inputs.</para></footnote>
+    inputs.)
     </para>
 
   </listitem>
diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 86c25d723..e1c589f61 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -26,7 +26,7 @@ is, in a normal (non-recursive) set, attributes are not added to the
 lexical scope; in a recursive set, they are.</para>
 
 <para>Recursive sets of course introduce the danger of infinite
-recursion.  For example,
+recursion.  For example, the expression
 
 <programlisting>
 rec {
@@ -34,9 +34,8 @@ rec {
   y = x;
 }.x</programlisting>
 
-does not terminate<footnote><para>Actually, Nix detects infinite
-recursion in this case and aborts (<quote>infinite recursion
-encountered</quote>).</para></footnote>.</para>
+will crash with an <literal>infinite recursion encountered</literal>
+error message.</para>
 
 </section>
 
diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml
index cebafb1f5..5520a4938 100644
--- a/doc/manual/expressions/language-values.xml
+++ b/doc/manual/expressions/language-values.xml
@@ -154,11 +154,10 @@ stdenv.mkDerivation {
 
   <listitem><para><emphasis>Paths</emphasis>, e.g.,
   <filename>/bin/sh</filename> or <filename>./builder.sh</filename>.
-  A path must contain at least one slash to be recognised as such; for
-  instance, <filename>builder.sh</filename> is not a
-  path<footnote><para>It's parsed as an expression that selects the
-  attribute <varname>sh</varname> from the variable
-  <varname>builder</varname>.</para></footnote>.  If the file name is
+  A path must contain at least one slash to be recognised as such. For
+  instance, <filename>builder.sh</filename> is not a path: it's parsed
+  as an expression that selects the attribute <varname>sh</varname>
+  from the variable <varname>builder</varname>. If the file name is
   relative, i.e., if it does not begin with a slash, it is made
   absolute at parse time relative to the directory of the Nix
   expression that contained it.  For instance, if a Nix expression in
@@ -176,7 +175,7 @@ stdenv.mkDerivation {
   <para>Paths can also be specified between angle brackets, e.g.
   <literal>&lt;nixpkgs&gt;</literal>. This means that the directories
   listed in the environment variable
-  <envar linkend="env-NIX_PATH">NIX_PATH</literal> will be searched
+  <literal linkend="env-NIX_PATH">NIX_PATH</literal> will be searched
   for the given file or directory name.
   </para>
 
diff --git a/doc/manual/expressions/simple-expression.xml b/doc/manual/expressions/simple-expression.xml
index 29fd872ee..ad97220a8 100644
--- a/doc/manual/expressions/simple-expression.xml
+++ b/doc/manual/expressions/simple-expression.xml
@@ -21,10 +21,9 @@ need to do three things:
   such as dependencies, sources, and so on.</para></listitem>
 
   <listitem><para>Write a <emphasis>builder</emphasis>.  This is a
-  shell script<footnote><para>In fact, it can be written in any
-  language, but typically it's a <command>bash</command> shell
-  script.</para></footnote> that actually builds the package from
-  the inputs.</para></listitem>
+  shell script that builds the package from the inputs. (In fact, it
+  can be written in any language, but typically it's a
+  <command>bash</command> shell script.)</para></listitem>
 
   <listitem><para>Add the package to the file
   <filename>pkgs/top-level/all-packages.nix</filename>.  The Nix
diff --git a/doc/manual/packages/profiles.xml b/doc/manual/packages/profiles.xml
index d7529d56c..3b8c5e293 100644
--- a/doc/manual/packages/profiles.xml
+++ b/doc/manual/packages/profiles.xml
@@ -18,13 +18,12 @@ of the Subversion package might be stored in a directory
 while another version might be stored in
 <filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
 The long strings prefixed to the directory names are cryptographic
-hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
-a base-32 notation, to be precise.</para></footnote> of
-<emphasis>all</emphasis> inputs involved in building the package —
-sources, dependencies, compiler flags, and so on.  So if two packages
-differ in any way, they end up in different locations in the file
-system, so they don’t interfere with each other.  Here is what a part
-of a typical Nix store looks like:</para>
+hashes (to be precise, 160-bit truncations of SHA-256 hashes encoded
+in a base-32 notation) of <emphasis>all</emphasis> inputs involved in
+building the package — sources, dependencies, compiler flags, and so
+on.  So if two packages differ in any way, they end up in different
+locations in the file system, so they don’t interfere with each other.
+Here is what a part of a typical Nix store looks like:</para>
 
 <mediaobject>
   <imageobject>
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 179fbf5ba..6c9f2d3e1 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -82,11 +82,10 @@ See also [???](#sec-common-options) for a list of common options.
 
   - `--find-file`  
     Look up the given files in Nix’s search path (as specified by the
-    NIX\_PATH\</literal\> environment variable). If found, print the
-    corresponding absolute paths on standard output. For instance, if
-    `NIX_PATH` is `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate
-    --find-file nixpkgs/default.nix` will print
-    `/home/alice/nixpkgs/default.nix`.
+    `NIX_PATH` environment variable). If found, print the corresponding
+    absolute paths on standard output. For instance, if `NIX_PATH` is
+    `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file
+    nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`.
 
   - `--strict`  
     When used with `--eval`, recursively evaluate list elements and
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index dce95773a..b9c65c81d 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -206,9 +206,9 @@ Most Nix commands accept the following command-line options:
 
   - `-I` *path*  
     Add a path to the Nix expression search path. This option may be
-    given multiple times. See the NIX\_PATH\</literal\> environment
-    variable for information on the semantics of the Nix search path.
-    Paths added through `-I` take precedence over `NIX_PATH`.
+    given multiple times. See the `NIX_PATH` environment variable for
+    information on the semantics of the Nix search path. Paths added
+    through `-I` take precedence over `NIX_PATH`.
 
   - `--option` *name* *value*  
     Set the Nix configuration option *name* to *value*. This overrides
diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md
index f922182c2..57222de85 100644
--- a/doc/manual/src/expressions/build-script.md
+++ b/doc/manual/src/expressions/build-script.md
@@ -19,10 +19,9 @@ steps to elucidate what a builder does. It performs the following steps:
 
 1.  When Nix runs a builder, it initially completely clears the
     environment (except for the attributes declared in the derivation).
-    For instance, the `PATH` variable is empty\[1\]. This is done to
-    prevent undeclared inputs from being used in the build process. If
-    for example the `PATH` contained `/usr/bin`, then you might
-    accidentally use `/usr/bin/gcc`.
+    This is done to prevent undeclared inputs from being used in the
+    build process. If for example the `PATH` contained `/usr/bin`, then
+    you might accidentally use `/usr/bin/gcc`.
     
     So the first step is to set up the environment. This is done by
     calling the `setup` script of the standard environment. The
@@ -67,6 +66,3 @@ If you are wondering about the absence of error checking on the result
 of various commands called in the builder: this is because the shell
 script is evaluated with Bash's `-e` option, which causes the script to
 be aborted if any command fails without an error check.
-
-1.  Actually, it's initialised to `/path-not-set` to prevent Bash from
-    setting it to a default value.
diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/expressions/derivations.md
index a4df20baa..4cb233e2e 100644
--- a/doc/manual/src/expressions/derivations.md
+++ b/doc/manual/src/expressions/derivations.md
@@ -5,11 +5,12 @@ describe a single derivation (a build action). It takes as input a set,
 the attributes of which specify the inputs of the build.
 
   - There must be an attribute named `system` whose value must be a
-    string specifying a Nix platform identifier, such as `"i686-linux"`
-    or `"x86_64-darwin"`\[1\] The build can only be performed on a
-    machine and operating system matching the platform identifier. (Nix
-    can automatically forward builds for other platforms by forwarding
-    them to other machines; see [???](#chap-distributed-builds).)
+    string specifying a Nix system type, such as `"i686-linux"` or
+    `"x86_64-darwin"`. (To figure out your system type, run `nix -vv
+    --version`.) The build can only be performed on a machine and
+    operating system matching the system type. (Nix can automatically
+    forward builds for other platforms by forwarding them to other
+    machines; see [???](#chap-distributed-builds).)
 
   - There must be an attribute named `name` whose value must be a
     string. This is used as a symbolic name for the package by
@@ -146,9 +147,3 @@ The builder is executed as follows:
     supported by Nix. This is because the Nix archives used in
     deployment have no concept of ownership information, and because it
     makes the build result dependent on the user performing the build.
-
-<!-- end list -->
-
-1.  To figure out your platform identifier, look at the line “Checking
-    for the canonical Nix system name” in the output of Nix's
-    `configure` script.
diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md
index 90bdc556b..6942abe70 100644
--- a/doc/manual/src/expressions/generic-builder.md
+++ b/doc/manual/src/expressions/generic-builder.md
@@ -27,8 +27,12 @@ Here is what each line means:
 1.  The `buildInputs` variable tells `setup` to use the indicated
     packages as “inputs”. This means that if a package provides a `bin`
     subdirectory, it's added to `PATH`; if it has a `include`
-    subdirectory, it's added to GCC's header search path; and so
-    on.\[1\]
+    subdirectory, it's added to GCC's header search path; and so on.
+    (This is implemented in a modular way: `setup` tries to source the
+    file `pkg/nix-support/setup-hook` of all dependencies. These “setup
+    hooks” can then set up whatever environment variables they want; for
+    instance, the setup hook for Perl sets the `PERL5LIB` environment
+    variable to contain the `lib/site_perl` directories of all inputs.)
 
 2.  The function `genericBuild` is defined in the file `$stdenv/setup`.
 
@@ -55,10 +59,3 @@ shorter:
 In fact, `mkDerivation` provides a default builder that looks exactly
 like that, so it is actually possible to omit the builder for Hello
 entirely.
-
-1.  How does it work? `setup` tries to source the file
-    `pkg/nix-support/setup-hook` of all dependencies. These “setup
-    hooks” can then set up whatever environment variables they want;
-    for instance, the setup hook for Perl sets the `PERL5LIB`
-    environment variable to contain the `lib/site_perl` directories of
-    all inputs.
diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/expressions/language-constructs.md
index 2699d675f..8e267a799 100644
--- a/doc/manual/src/expressions/language-constructs.md
+++ b/doc/manual/src/expressions/language-constructs.md
@@ -17,14 +17,14 @@ would be invalid if no such variable exists. That is, in a normal
 recursive set, they are.
 
 Recursive sets of course introduce the danger of infinite recursion. For
-example,
+example, the expression
 
     rec {
       x = y;
       y = x;
     }.x
 
-does not terminate\[1\].
+will crash with an `infinite recursion encountered` error message.
 
 ## Let-expressions
 
@@ -304,6 +304,3 @@ establishes the same scope as
 Comments can be single-line, started with a `#` character, or
 inline/multi-line, enclosed within `/*
 ... */`.
-
-1.  Actually, Nix detects infinite recursion in this case and aborts
-    (“infinite recursion encountered”).
diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md
index 8ffe2f0f9..eca2cab51 100644
--- a/doc/manual/src/expressions/language-values.md
+++ b/doc/manual/src/expressions/language-values.md
@@ -108,12 +108,14 @@ Nix has the following basic data types:
     floating point number will have a floating point number as a result.
 
   - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at
-    least one slash to be recognised as such; for instance, `builder.sh`
-    is not a path\[1\]. If the file name is relative, i.e., if it does
-    not begin with a slash, it is made absolute at parse time relative
-    to the directory of the Nix expression that contained it. For
-    instance, if a Nix expression in `/foo/bar/bla.nix` refers to
-    `../xyzzy/fnord.nix`, the absolute path is `/foo/xyzzy/fnord.nix`.
+    least one slash to be recognised as such. For instance, `builder.sh`
+    is not a path: it's parsed as an expression that selects the
+    attribute `sh` from the variable `builder`. If the file name is
+    relative, i.e., if it does not begin with a slash, it is made
+    absolute at parse time relative to the directory of the Nix
+    expression that contained it. For instance, if a Nix expression in
+    `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path
+    is `/foo/xyzzy/fnord.nix`.
     
     If the first component of a path is a `~`, it is interpreted as if
     the rest of the path were relative to the user's home directory.
@@ -122,8 +124,8 @@ Nix has the following basic data types:
     
     Paths can also be specified between angle brackets, e.g.
     `<nixpkgs>`. This means that the directories listed in the
-    environment variable NIX\_PATH\</literal\> will be searched for the
-    given file or directory name.
+    environment variable `NIX_PATH` will be searched for the given file
+    or directory name.
 
   - *Booleans* with values `true` and `false`.
 
@@ -210,6 +212,3 @@ passed in first , e.g.,
 evaluates to `2`. This can be used to attach metadata to a function
 without the caller needing to treat it specially, or to implement a form
 of object-oriented programming, for example.
-
-1.  It's parsed as an expression that selects the attribute `sh` from
-    the variable `builder`.
diff --git a/doc/manual/src/expressions/simple-expression.md b/doc/manual/src/expressions/simple-expression.md
index 9023f97a4..857f71b9b 100644
--- a/doc/manual/src/expressions/simple-expression.md
+++ b/doc/manual/src/expressions/simple-expression.md
@@ -12,16 +12,12 @@ do three things:
     describes all the inputs involved in building the package, such as
     dependencies, sources, and so on.
 
-2.  Write a *builder*. This is a shell script\[1\] that actually builds
-    the package from the inputs.
+2.  Write a *builder*. This is a shell script that builds the package
+    from the inputs. (In fact, it can be written in any language, but
+    typically it's a `bash` shell script.)
 
 3.  Add the package to the file `pkgs/top-level/all-packages.nix`. The
     Nix expression written in the first step is a *function*; it
     requires other packages in order to build it. In this step you put
     it all together, i.e., you call the function with the right
     arguments to build the actual package.
-
-<!-- end list -->
-
-1.  In fact, it can be written in any language, but typically it's a
-    `bash` shell script.
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
index 9076033d7..1950c3121 100644
--- a/doc/manual/src/package-management/profiles.md
+++ b/doc/manual/src/package-management/profiles.md
@@ -10,12 +10,13 @@ in a directory
 `/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/`, while
 another version might be stored in
 `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long
-strings prefixed to the directory names are cryptographic hashes\[1\] of
-*all* inputs involved in building the package — sources, dependencies,
-compiler flags, and so on. So if two packages differ in any way, they
-end up in different locations in the file system, so they don’t
-interfere with each other. Here is what a part of a typical Nix store
-looks like:
+strings prefixed to the directory names are cryptographic hashes (to be
+precise, 160-bit truncations of SHA-256 hashes encoded in a base-32
+notation) of *all* inputs involved in building the package — sources,
+dependencies, compiler flags, and so on. So if two packages differ in
+any way, they end up in different locations in the file system, so they
+don’t interfere with each other. Here is what a part of a typical Nix
+store looks like:
 
 ![](../figures/user-environments.png)
 
@@ -113,6 +114,3 @@ All `nix-env` operations work on the profile pointed to by
     $ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
 
 This will *not* change the `~/.nix-profile` symlink.
-
-1.  160-bit truncations of SHA-256 hashes encoded in a base-32 notation,
-    to be precise.

From 758c9ee1bb0e9d4bea420420af93e0128fabf188 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 12:56:19 +0200
Subject: [PATCH 040/882] Clean up the manpages

---
 doc/manual/src/command-ref/conf-file.md       |  12 +-
 doc/manual/src/command-ref/nix-build.md       |  54 +---
 doc/manual/src/command-ref/nix-channel.md     |  36 +--
 .../src/command-ref/nix-collect-garbage.md    |  26 +-
 .../src/command-ref/nix-copy-closure.md       |   9 +-
 doc/manual/src/command-ref/nix-daemon.md      |  12 +-
 doc/manual/src/command-ref/nix-env.md         | 284 +++++-------------
 doc/manual/src/command-ref/nix-hash.md        |  50 +--
 doc/manual/src/command-ref/nix-instantiate.md |  76 ++---
 .../src/command-ref/nix-prefetch-url.md       |  38 +--
 doc/manual/src/command-ref/nix-shell.md       | 103 ++-----
 doc/manual/src/command-ref/nix-store.md       | 249 +++------------
 12 files changed, 248 insertions(+), 701 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
index 92abeb73e..4fa9cedfc 100644
--- a/doc/manual/src/command-ref/conf-file.md
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -1,12 +1,8 @@
-nix.conf
+Title: nix.conf
 
-5
+# Name
 
-Nix
-
-nix.conf
-
-Nix configuration file
+`nix.conf` - Nix configuration file
 
 # Description
 
@@ -20,7 +16,7 @@ By default Nix reads settings from the following places:
 
   - If `NIX_USER_CONF_FILES` is set, then each path separated by `:`
     will be loaded in reverse order.
-    
+
     Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
     and `XDG_CONFIG_HOME`. If these are unset, it will look in
     `$HOME/.config/nix.conf`.
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index ddebc4b1b..a65f53b4b 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -1,44 +1,18 @@
-nix-build
+Title: nix-build
 
-1
+# Name
 
-Nix
+`nix-build` - build a Nix expression
 
-nix-build
+# Synopsis
 
-build a Nix expression
-
-nix-build
-
-\--arg
-
-name
-
-value
-
-\--argstr
-
-name
-
-value
-
-\--attr
-
-\-A
-
-attrPath
-
-\--no-out-link
-
-\--dry-run
-
-\--out-link
-
-\-o
-
-outlink
-
-paths
+`nix-build` [*paths…*]
+  [`--arg` *name* *value*]
+  [`--argstr` *name* *value*]
+  [{`--attr` | `-A`} *attrPath*]
+  [`--no-out-link`]
+  [`--dry-run`]
+  [{`--out-link` | `-o`} *outlink*]
 
 # Description
 
@@ -63,7 +37,7 @@ expression to a low-level store derivation) and [`nix-store
 --realise`](#rsec-nix-store-realise) (to build the store derivation).
 
 > **Warning**
-> 
+>
 > The result of the build is automatically registered as a root of the
 > Nix garbage collector. This root disappears automatically when the
 > `result` symlink is deleted or renamed. So don’t rename the symlink.
@@ -94,10 +68,10 @@ The following common options are supported:
     $ nix-build '<nixpkgs>' -A firefox
     store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
     /nix/store/d18hyl92g30l...-firefox-1.5.0.7
-    
+
     $ ls -l result
     lrwxrwxrwx  ...  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
-    
+
     $ ls ./result/bin/
     firefox  firefox-config
 
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index 0c20788f0..6b4bd2459 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -1,34 +1,12 @@
-nix-channel
+Title: nix-channel
 
-1
+# Name
 
-Nix
+`nix-channel` - manage Nix channels
 
-nix-channel
+# Synopsis
 
-manage Nix channels
-
-nix-channel
-
-\--add
-
-url
-
-name
-
-\--remove
-
-name
-
-\--list
-
-\--update
-
-names
-
-\--rollback
-
-generation
+`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] }
 
 # Description
 
@@ -82,10 +60,10 @@ You can revert channel updates using `--rollback`:
 
     $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
     "14.04.527.0e935f1"
-    
+
     $ nix-channel --rollback
     switching from generation 483 to 482
-    
+
     $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
     "14.04.526.dbadfad"
 
diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index 77b5d42d3..7b246f3b3 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -1,28 +1,12 @@
-nix-collect-garbage
+Title: nix-collect-garbage
 
-1
+# Name
 
-Nix
+`nix-collect-garbage` - delete unreachable store paths
 
-nix-collect-garbage
+# Synopsis
 
-delete unreachable store paths
-
-nix-collect-garbage
-
-\--delete-old
-
-\-d
-
-\--delete-older-than
-
-period
-
-\--max-freed
-
-bytes
-
-\--dry-run
+`nix-collect-garbage` [`--delete-old`] [`-d`] [`--delete-older-than` *period*] [`--max-freed` *bytes*] [`--dry-run`]
 
 # Description
 
diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index 037334c4d..4026e50de 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -6,7 +6,13 @@ Title: nix-copy-closure
 
 # Synopsis
 
-`nix-copy-closure` [`--to` | `--from`] [`--gzip`] [`--include-outputs`] [`--use-substitutes` | `-s`] [`-v`] _user@machine_ _paths_
+`nix-copy-closure`
+  [`--to` | `--from`]
+  [`--gzip`]
+  [`--include-outputs`]
+  [`--use-substitutes` | `-s`]
+  [`-v`]
+  _user@machine_ _paths_
 
 # Description
 
@@ -75,4 +81,3 @@ environment:
     $ nix-copy-closure --from alice@itchy.labs \
         /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
     $ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
-
diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/src/command-ref/nix-daemon.md
index b0570789c..bd5d25026 100644
--- a/doc/manual/src/command-ref/nix-daemon.md
+++ b/doc/manual/src/command-ref/nix-daemon.md
@@ -1,14 +1,12 @@
-nix-daemon
+Title: nix-daemon
 
-8
+# Name
 
-Nix
+`nix-daemon` - Nix multi-user support daemon
 
-nix-daemon
+# Synopsis
 
-Nix multi-user support daemon
-
-nix-daemon
+`nix-daemon`
 
 # Description
 
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index 90a2bd351..cf688100e 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -1,50 +1,20 @@
-nix-env
+Title: nix-env
 
-1
+# Name
 
-Nix
+`nix-env` - manipulate or query Nix user environments
 
-nix-env
+# Synopsis
 
-manipulate or query Nix user environments
-
-nix-env
-
-\--arg
-
-name
-
-value
-
-\--argstr
-
-name
-
-value
-
-\--file
-
-\-f
-
-path
-
-\--profile
-
-\-p
-
-path
-
-\--system-filter
-
-system
-
-\--dry-run
-
-operation
-
-options
-
-arguments
+`nix-env`
+  [`--option` *name* *value*]
+  [`--arg` *name* *value*]
+  [`--argstr` *name* *value*]
+  [{`--file` | `-f`} *path*]
+  [{`--profile` | `-p`} *path(]
+  [`--system-filter` *system*]
+  [`--dry-run`]
+  *operation* [*options…*] [*arguments…*]
 
 # Description
 
@@ -103,7 +73,7 @@ have an effect. See also [???](#sec-common-options).
     expression*) used by the `--install`, `--upgrade`, and `--query
     --available` operations to obtain derivations. The default is
     `~/.nix-defexpr`.
-    
+
     If the argument starts with `http://` or `https://`, it is
     interpreted as the URL of a tarball that will be downloaded and
     unpacked to a temporary location. The tarball must include a single
@@ -120,7 +90,7 @@ have an effect. See also [???](#sec-common-options).
     `--switch-generation`, `--delete-generations` and `--rollback`
     operations, this flag will cause `nix-env` to print what *would* be
     done if this flag had not been specified, without actually doing it.
-    
+
     `--dry-run` also prints out which paths will be
     [substituted](#gloss-substitute) (i.e., downloaded) and which paths
     will be built from source (because no substitute is available).
@@ -135,38 +105,38 @@ have an effect. See also [???](#sec-common-options).
 # Files
 
   - `~/.nix-defexpr`  
-    The source for the default Nix expressions used by the `--install`,
-    `--upgrade`, and `--query
-                    --available` operations to obtain derivations. The `--file` option
-    may be used to override this default.
-    
+    The source for the default Nix expressions used by the
+    `--install`, `--upgrade`, and `--query --available` operations to
+    obtain derivations. The `--file` option may be used to override
+    this default.
+
     If `~/.nix-defexpr` is a file, it is loaded as a Nix expression. If
     the expression is a set, it is used as the default Nix expression.
     If the expression is a function, an empty set is passed as argument
     and the return value is used as the default Nix expression.
-    
+
     If `~/.nix-defexpr` is a directory containing a `default.nix` file,
     that file is loaded as in the above paragraph.
-    
+
     If `~/.nix-defexpr` is a directory without a `default.nix` file,
     then its contents (both files and subdirectories) are loaded as Nix
     expressions. The expressions are combined into a single set, each
     expression under an attribute with the same name as the original
     file or subdirectory.
-    
+
     For example, if `~/.nix-defexpr` contains two files, `foo.nix` and
     `bar.nix`, then the default Nix expression will essentially be
-    
+
         {
           foo = import ~/.nix-defexpr/foo.nix;
           bar = import ~/.nix-defexpr/bar.nix;
         }
-    
+
     The file `manifest.nix` is always ignored. Subdirectories without a
     `default.nix` file are traversed recursively in search of more Nix
     expressions, but the names of these intermediate directories are not
     added to the attribute paths of the default Nix expression.
-    
+
     The command `nix-channel` places symlinks to the downloaded Nix
     expressions from each subscribed channel in this directory.
 
@@ -180,21 +150,13 @@ have an effect. See also [???](#sec-common-options).
 
 ## Synopsis
 
-nix-env
-
-\--install
-
-\-i
-
-\--preserve-installed
-
-\-P
-
-\--remove-all
-
-\-r
-
-args
+`nix-env` {`--install` | `-i`} *args…*
+  [{`--prebuilt-only` | `-b`}]
+  [{`--attr` | `-A`}]
+  [`--from-expression`] [`-E`]
+  [`--from-profile` *path*]
+  [`--preserve-installed` | `-P`]
+  [`--remove-all` | `-r`]
 
 ## Description
 
@@ -208,17 +170,17 @@ a number of possible ways:
     output paths are installed. Currently installed derivations with a
     name equal to the name of a derivation being added are removed
     unless the option `--preserve-installed` is specified.
-    
+
     If there are multiple derivations matching a name in *args* that
     have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the
     derivation with the highest *priority* is used. A derivation can
     define a priority by declaring the `meta.priority` attribute. This
     attribute should be a number, with a higher value denoting a lower
     priority. The default priority is `0`.
-    
+
     If there are multiple matching derivations with the same priority,
     then the derivation with the highest version will be installed.
-    
+
     You can force the installation of multiple derivations with the same
     name by being specific about the versions. For instance, `nix-env -i
     gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will
@@ -339,21 +301,13 @@ channel:
 
 ## Synopsis
 
-nix-env
-
-\--upgrade
-
-\-u
-
-\--lt
-
-\--leq
-
-\--eq
-
-\--always
-
-args
+`nix-env` {`--upgrade` | `-u`} *args*
+  [`--lt` | `--leq` | `--eq` | `--always`]
+  [{`--prebuilt-only` | `-b`}]
+  [{`--attr` | `-A`}]
+  [`--from-expression`] [`-E`]
+  [`--from-profile` *path*]
+  [`--preserve-installed` | `-P`]
 
 ## Description
 
@@ -400,13 +354,13 @@ For the other flags, see `--install`.
 
     $ nix-env --upgrade gcc
     upgrading `gcc-3.3.1' to `gcc-3.4'
-    
+
     $ nix-env -u gcc-3.3.2 --always (switch to a specific version)
     upgrading `gcc-3.4' to `gcc-3.3.2'
-    
+
     $ nix-env --upgrade pan
     (no upgrades available, so nothing happens)
-    
+
     $ nix-env -u (try to upgrade everything)
     upgrading `hello-2.1.2' to `hello-2.1.3'
     upgrading `mozilla-1.2' to `mozilla-1.4'
@@ -451,19 +405,13 @@ This is illustrated by the following examples:
 
 ## Synopsis
 
-nix-env
-
-\--uninstall
-
-\-e
-
-drvnames
+`nix-env` {`--uninstall` | `-e`} *drvnames…*
 
 ## Description
 
 The uninstall operation creates a new user environment, based on the
 current generation of the active profile, from which the store paths
-designated by the symbolic names *names* are removed.
+designated by the symbolic names *drvnames* are removed.
 
 ## Examples
 
@@ -474,11 +422,7 @@ designated by the symbolic names *names* are removed.
 
 ## Synopsis
 
-nix-env
-
-\--set
-
-drvname
+`nix-env` `--set` *drvname*
 
 ## Description
 
@@ -496,15 +440,7 @@ contain just Firefox:
 
 ## Synopsis
 
-nix-env
-
-\--set-flag
-
-name
-
-value
-
-drvnames
+`nix-env` `--set-flag` *name* *value* *drvnames*
 
 ## Description
 
@@ -545,20 +481,20 @@ while the old remains part of the profile:
 
     $ nix-env -q
     firefox-2.0.0.9 (the current one)
-    
+
     $ nix-env --preserve-installed -i firefox-2.0.0.11
     installing `firefox-2.0.0.11'
     building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
     collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox'
       and `/nix/store/...-firefox-2.0.0.9/bin/firefox'.
     (i.e., can’t have two active at the same time)
-    
+
     $ nix-env --set-flag active false firefox
     setting flag on `firefox-2.0.0.9'
-    
+
     $ nix-env --preserve-installed -i firefox-2.0.0.11
     installing `firefox-2.0.0.11'
-    
+
     $ nix-env -q
     firefox-2.0.0.11 (the enabled one)
     firefox-2.0.0.9 (the disabled one)
@@ -572,57 +508,21 @@ To make files from `binutils` take precedence over files from `gcc`:
 
 ## Synopsis
 
-nix-env
-
-\--query
-
-\-q
-
-\--installed
-
-\--available
-
-\-a
-
-\--status
-
-\-s
-
-\--attr-path
-
-\-P
-
-\--no-name
-
-\--compare-versions
-
-\-c
-
-\--system
-
-\--drv-path
-
-\--out-path
-
-\--description
-
-\--meta
-
-\--xml
-
-\--json
-
-\--prebuilt-only
-
-\-b
-
-\--attr
-
-\-A
-
-attribute-path
-
-names
+`nix-env` {`--query` | `-q`} *names…*
+  [`--installed` | `--available` | `-a`]
+  [{`--status` | `-s`}]
+  [{`--attr-path` | `-P`}]
+  [`--no-name`]
+  [{`--compare-versions` | `-c`}]
+  [`--system`]
+  [`--drv-path`]
+  [`--out-path`]
+  [`--description`]
+  [`--meta`]
+  [`--xml`]
+  [`--json`]
+  [{`--prebuilt-only` | `-b`}]
+  [{`--attr` | `-A`} *attribute-path*]
 
 ## Description
 
@@ -696,17 +596,17 @@ derivation is shown unless `--no-name` is specified.
     `--available` is given). This is useful for quickly seeing whether
     upgrades for installed packages are available in a Nix expression. A
     column is added with the following meaning:
-    
+
       - `<` *version*  
         A newer version of the package is available or installed.
-    
+
       - `=` *version*  
         At most the same version of the package is available or
         installed.
-    
+
       - `>` *version*  
         Only older versions of the package are available or installed.
-    
+
       - `- ?`  
         No version of the package is available or installed.
 
@@ -797,13 +697,7 @@ To show all packages in the latest revision of the Nixpkgs repository:
 
 ## Synopsis
 
-nix-env
-
-\--switch-profile
-
-\-S
-
-path
+`nix-env` {`--switch-profile` | `-S`} *path*
 
 ## Description
 
@@ -818,9 +712,7 @@ the symlink `~/.nix-profile` is made to point to *path*.
 
 ## Synopsis
 
-nix-env
-
-\--list-generations
+`nix-env` `--list-generations`
 
 ## Description
 
@@ -841,11 +733,7 @@ generation, and indicates the current generation.
 
 ## Synopsis
 
-nix-env
-
-\--delete-generations
-
-generations
+`nix-env` `--delete-generations` *generations*
 
 ## Description
 
@@ -862,24 +750,18 @@ generations is important to make garbage collection effective.
 ## Examples
 
     $ nix-env --delete-generations 3 4 8
-    
+
     $ nix-env --delete-generations +5
-    
+
     $ nix-env --delete-generations 30d
-    
+
     $ nix-env -p other_profile --delete-generations old
 
 # Operation `--switch-generation`
 
 ## Synopsis
 
-nix-env
-
-\--switch-generation
-
-\-G
-
-generation
+`nix-env` {`--switch-generation` | `-G`} *generation*
 
 ## Description
 
@@ -900,9 +782,7 @@ Switching will fail if the specified generation does not exist.
 
 ## Synopsis
 
-nix-env
-
-\--rollback
+`nix-env` `--rollback`
 
 ## Description
 
@@ -915,7 +795,7 @@ generation, if it exists. It is just a convenience wrapper around
 
     $ nix-env --rollback
     switching from generation 92 to 91
-    
+
     $ nix-env --rollback
     error: no generation older than the current (91) exists
 
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index 3b8bbf740..38c9e0f67 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -1,38 +1,16 @@
-nix-hash
+Title: nix-hash
 
-1
+# Name
 
-Nix
+`nix-hash` - compute the cryptographic hash of a path
 
-nix-hash
+# Synopsis
 
-compute the cryptographic hash of a path
+`nix-hash` [`--flat`] [`--base32`] [`--truncate`] [`--type` *hashAlgo*] *path…*
 
-nix-hash
+`nix-hash` `--to-base16` *hash…*
 
-\--flat
-
-\--base32
-
-\--truncate
-
-\--type
-
-hashAlgo
-
-path
-
-nix-hash
-
-\--to-base16
-
-hash
-
-nix-hash
-
-\--to-base32
-
-hash
+`nix-hash` `--to-base32` *hash…*
 
 # Description
 
@@ -92,22 +70,22 @@ Computing hashes:
 
     $ mkdir test
     $ echo "hello" > test/world
-    
+
     $ nix-hash test/ (MD5 hash; default)
     8179d3caeff1869b5ba1744e5a245c04
-    
+
     $ nix-store --dump test/ | md5sum (for comparison)
     8179d3caeff1869b5ba1744e5a245c04  -
-    
+
     $ nix-hash --type sha1 test/
     e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
-    
+
     $ nix-hash --type sha1 --base32 test/
     nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
-    
+
     $ nix-hash --type sha256 --flat test/
     error: reading file `test/': Is a directory
-    
+
     $ nix-hash --type sha256 --flat test/world
     5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03
 
@@ -115,6 +93,6 @@ Converting between hexadecimal and base-32:
 
     $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
     nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
-    
+
     $ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
     e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 6c9f2d3e1..b6bbbe80a 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -1,56 +1,22 @@
-nix-instantiate
+Title: nix-instantiate
 
-1
+# Name
 
-Nix
+`nix-instantiate` - instantiate store derivations from Nix expressions
 
-nix-instantiate
+# Synopsis
 
-instantiate store derivations from Nix expressions
+`nix-instantiate`
+  [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ]
+  [`--read-write-mode`]
+  [`--arg` *name* *value*]
+  [{`--attr`| `-A`} *attrPath*]
+  [`--add-root` *path*]
+  [`--indirect`]
+  [`--expr` | `-E`]
+  *files…*
 
-nix-instantiate
-
-\--parse
-
-\--eval
-
-\--strict
-
-\--json
-
-\--xml
-
-\--read-write-mode
-
-\--arg
-
-name
-
-value
-
-\--attr
-
-\-A
-
-attrPath
-
-\--add-root
-
-path
-
-\--indirect
-
-\--expr
-
-\-E
-
-files
-
-nix-instantiate
-
-\--find-file
-
-files
+`nix-instantiate` `--find-file` *files…*
 
 # Description
 
@@ -91,9 +57,9 @@ See also [???](#sec-common-options) for a list of common options.
     When used with `--eval`, recursively evaluate list elements and
     attributes. Normally, such sub-expressions are left unevaluated
     (since the Nix expression language is lazy).
-    
+
     > **Warning**
-    > 
+    >
     > This option can cause non-termination, because lazy data
     > structures can be infinitely large.
 
@@ -123,11 +89,11 @@ using `nix-store`:
 
     $ nix-instantiate test.nix (instantiate)
     /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
-    
+
     $ nix-store -r $(nix-instantiate test.nix) (build)
     ...
     /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path)
-    
+
     $ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
     dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
     ...
@@ -145,10 +111,10 @@ Parsing and evaluating Nix expressions:
 
     $ nix-instantiate --parse -E '1 + 2'
     1 + 2
-    
+
     $ nix-instantiate --eval -E '1 + 2'
     3
-    
+
     $ nix-instantiate --eval --xml -E '1 + 2'
     <?xml version='1.0' encoding='utf-8'?>
     <expr>
@@ -179,5 +145,3 @@ attempt to show non-normal forms).
         <string value="foo" />
       </attr>
     ...
-
-# Environment variables
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index f7de27402..688496969 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -1,32 +1,16 @@
-nix-prefetch-url
+Title: nix-prefetch-url
 
-1
+# Name
 
-Nix
+`nix-prefetch-url` - copy a file from a URL into the store and print its hash
 
-nix-prefetch-url
+# Synopsis
 
-copy a file from a URL into the store and print its hash
-
-nix-prefetch-url
-
-\--version
-
-\--type
-
-hashAlgo
-
-\--print-path
-
-\--unpack
-
-\--name
-
-name
-
-url
-
-hash
+`nix-prefetch-url` *url* [*hash*]
+  [`--type` *hashAlgo*]
+  [`--print-path`]
+  [`--unpack`]
+  [`--name` *name*]
 
 # Description
 
@@ -77,11 +61,11 @@ Nix store is also printed.
 
     $ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
     0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
-    
+
     $ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
     0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
     /nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
-    
+
     $ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz
     079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7
     /nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 9e2b781ab..d6dbb6e26 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -1,60 +1,21 @@
-nix-shell
+Title: nix-shell
 
-1
+# Name
 
-Nix
+`nix-shell` - start an interactive shell based on a Nix expression
 
-nix-shell
+# Synopsis
 
-start an interactive shell based on a Nix expression
-
-nix-shell
-
-\--arg
-
-name
-
-value
-
-\--argstr
-
-name
-
-value
-
-\--attr
-
-\-A
-
-attrPath
-
-\--command
-
-cmd
-
-\--run
-
-cmd
-
-\--exclude
-
-regexp
-
-\--pure
-
-\--keep
-
-name
-
-\--packages
-
-\-p
-
-packages
-
-expressions
-
-path
+`nix-shell`
+  [`--arg` *name* *value*]
+  [`--argstr` *name* *value*]
+  [{`--attr` | `-A`} *attrPath*]
+  [`--command` *cmd*]
+  [`--run` *cmd*]
+  [`--exclude` *regexp*]
+  [--pure]
+  [--keep *name*]
+  {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]}
 
 # Description
 
@@ -96,11 +57,10 @@ All options not listed here are passed to `nix-store
     This command is executed in an interactive shell. (Use `--run` to
     use a non-interactive shell instead.) However, a call to `exit` is
     implicitly added to the command, so the shell will exit after
-    running the command. To prevent this, add `return` at the end; e.g.
-    `--command
-                    "echo Hello; return"` will print `Hello` and then drop you into the
-    interactive shell. This can be useful for doing any additional
-    initialisation.
+    running the command. To prevent this, add `return` at the end;
+    e.g.  `--command "echo Hello; return"` will print `Hello` and then
+    drop you into the interactive shell. This can be useful for doing
+    any additional initialisation.
 
   - `--run` *cmd*  
     Like `--command`, but executes the command in a non-interactive
@@ -129,8 +89,7 @@ All options not listed here are passed to `nix-store
 
   - `-i` *interpreter*  
     The chained script interpreter to be invoked by `nix-shell`. Only
-    applicable in `#!`-scripts (described
-    [below](#ssec-nix-shell-shebang)).
+    applicable in `#!`-scripts (described below).
 
   - `--keep` *name*  
     When a `--pure` shell is started, keep the listed environment
@@ -186,7 +145,7 @@ gives you a shell containing the Pan package from a specific revision of
 Nixpkgs:
 
     $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
-    
+
     [nix-shell:~]$ pan --version
     Pan 0.139
 
@@ -213,9 +172,9 @@ For example, here is a Python script that depends on Python and the
 
     #! /usr/bin/env nix-shell
     #! nix-shell -i python -p python pythonPackages.prettytable
-    
+
     import prettytable
-    
+
     # Print a simple table.
     t = prettytable.PrettyTable(["N", "N^2"])
     for n in range(1, 10): t.add_row([n, n * n])
@@ -226,12 +185,12 @@ requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages:
 
     #! /usr/bin/env nix-shell
     #! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
-    
+
     use HTML::TokeParser::Simple;
-    
+
     # Fetch nixos.org and print all hrefs.
     my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
-    
+
     while (my $token = $p->get_tag("a")) {
         my $href = $token->get_attr("href");
         print "$href\n" if $href;
@@ -242,11 +201,11 @@ package like Terraform:
 
     #! /usr/bin/env nix-shell
     #! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
-    
+
     terraform apply
 
 > **Note**
-> 
+>
 > You must use double quotes (`"`) when passing a simple Nix expression
 > in a nix-shell shebang.
 
@@ -257,10 +216,10 @@ branch):
     #! /usr/bin/env nix-shell
     #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
     #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
-    
+
     import Network.HTTP
     import Text.HTML.TagSoup
-    
+
     -- Fetch nixos.org and print all hrefs.
     main = do
       resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
@@ -285,7 +244,5 @@ where the file `deps.nix` in the same directory as the `#!`-script
 contains:
 
     with import <nixpkgs> {};
-    
-    runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
 
-# Environment variables
+    runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index a666de49b..a098beb1e 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -1,26 +1,15 @@
-nix-store
+Title: nix-store
 
-1
+# Name
 
-Nix
+`nix-store` - manipulate or query the Nix store
 
-nix-store
+# Synopsis
 
-manipulate or query the Nix store
-
-nix-store
-
-\--add-root
-
-path
-
-\--indirect
-
-operation
-
-options
-
-arguments
+`nix-store` *operation* [*options…*] [*arguments…*]
+  [`--option` *name* *value*]
+  [`--add-root` *path*]
+  [`--indirect`]
 
 # Description
 
@@ -44,7 +33,7 @@ options.
     which must be inside a directory that is scanned for roots by the
     garbage collector (i.e., typically in a subdirectory of
     `/nix/var/nix/gcroots/`) *unless* the `--indirect` flag is used.
-    
+
     If there are multiple results, then multiple symlinks will be
     created by sequentially numbering symlinks beyond the first one
     (e.g., `foo`, `foo-2`, `foo-3`, and so on).
@@ -55,24 +44,24 @@ options.
     commands such as `nix-build` that place a symlink to the build
     result in the current directory; such a build result should not be
     garbage-collected unless the symlink is removed.
-    
+
     The `--indirect` flag causes a uniquely named symlink to *path* to
     be stored in `/nix/var/nix/gcroots/auto/`. For instance,
-    
+
         $ nix-store --add-root /home/eelco/bla/result --indirect -r ...
-        
+
         $ ls -l /nix/var/nix/gcroots/auto
         lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
-        
+
         $ ls -l /home/eelco/bla/result
         lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10
-    
+
     Thus, when `/home/eelco/bla/result` is removed, the GC root in the
     `auto` directory becomes a dangling symlink and will be ignored by
     the collector.
-    
+
     > **Warning**
-    > 
+    >
     > Note that it is not possible to move or rename indirect GC roots,
     > since the symlink in the `auto` directory will still point to the
     > old location.
@@ -83,15 +72,7 @@ options.
 
 ## Synopsis
 
-nix-store
-
-\--realise
-
-\-r
-
-paths
-
-\--dry-run
+`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`]
 
 ## Description
 
@@ -136,7 +117,7 @@ The following flags are available:
     output path is not identical to the corresponding output from the
     previous build, the new output path is left in
     `/nix/store/name.check.`
-    
+
     See also the `build-repeat` configuration option, which repeats a
     derivation a number of times and prevents its outputs from being
     registered as “valid” in the Nix store unless they are identical.
@@ -188,11 +169,7 @@ To test whether a previously-built derivation is deterministic:
 
 ## Synopsis
 
-nix-store
-
-\--serve
-
-\--write
+`nix-store` `--serve` [`--write`]
 
 ## Description
 
@@ -220,19 +197,7 @@ used to provide build access to a given SSH public key:
 
 ## Synopsis
 
-nix-store
-
-\--gc
-
-\--print-roots
-
-\--print-live
-
-\--print-dead
-
-\--max-freed
-
-bytes
+`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*]
 
 ## Description
 
@@ -294,13 +259,7 @@ To delete at least 100 MiBs of unreachable paths:
 
 ## Synopsis
 
-nix-store
-
-\--delete
-
-\--ignore-liveness
-
-paths
+`nix-store` `--delete` [`--ignore-liveness`] *paths…*
 
 ## Description
 
@@ -324,55 +283,13 @@ paths in the store that refer to it (i.e., depend on it).
 
 ## Synopsis
 
-nix-store
-
-\--query
-
-\-q
-
-\--outputs
-
-\--requisites
-
-\-R
-
-\--references
-
-\--referrers
-
-\--referrers-closure
-
-\--deriver
-
-\-d
-
-\--graph
-
-\--tree
-
-\--binding
-
-name
-
-\-b
-
-name
-
-\--hash
-
-\--size
-
-\--roots
-
-\--use-output
-
-\-u
-
-\--force-realise
-
-\-f
-
-paths
+`nix-store` {`--query` | `-q`}
+  {`--outputs` | `--requisites` | `-R` | `--references` |
+  `--referrers` | `--referrers-closure` | `--deriver` | `-d` |
+  `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` |
+  `--size` | `--roots`}
+  [`--use-output`] [`-u`] [`--force-realise`] [`-f`]
+  *paths…*
 
 ## Description
 
@@ -403,13 +320,13 @@ symlink.
 
   - `--requisites`; `-R`  
     Prints out the [closure](#gloss-closure) of the store path *paths*.
-    
+
     This query has one option:
-    
-      - `--include-outputs`  
+
+      - `--include-outputs`
         Also include the output path of store derivations, and their
         closures.
-    
+
     This query can be used to implement various kinds of deployment. A
     *source deployment* is obtained by distributing the closure of a
     store derivation. A *binary deployment* is obtained by distributing
@@ -555,11 +472,7 @@ depends on `svn`:
 
 ## Synopsis
 
-nix-store
-
-\--add
-
-paths
+`nix-store` `--add` *paths…*
 
 ## Description
 
@@ -575,15 +488,7 @@ prints the resulting paths in the Nix store on standard output.
 
 ## Synopsis
 
-nix-store
-
-\--recursive
-
-\--add-fixed
-
-algorithm
-
-paths
+`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…*
 
 ## Description
 
@@ -608,13 +513,7 @@ This operation has the following options:
 
 ## Synopsis
 
-nix-store
-
-\--verify
-
-\--check-contents
-
-\--repair
+`nix-store` `--verify` [`--check-contents`] [`--repair`]
 
 ## Description
 
@@ -643,11 +542,7 @@ This operation has the following options:
 
 ## Synopsis
 
-nix-store
-
-\--verify-path
-
-paths
+`nix-store` `--verify-path` *paths…*
 
 ## Description
 
@@ -666,11 +561,7 @@ To verify the integrity of the `svn` command and all its dependencies:
 
 ## Synopsis
 
-nix-store
-
-\--repair-path
-
-paths
+`nix-store` `--repair-path` *paths…*
 
 ## Description
 
@@ -679,7 +570,7 @@ by redownloading them using the available substituters. If no
 substitutes are available, then repair is not possible.
 
 > **Warning**
-> 
+>
 > During repair, there is a very small time window during which the old
 > path (if it exists) is moved out of the way and replaced with the new
 > path. If repair is interrupted in between, then the system may be left
@@ -692,7 +583,7 @@ substitutes are available, then repair is not possible.
     path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified!
       expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588',
       got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4'
-    
+
     $ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
     fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
     …
@@ -701,11 +592,7 @@ substitutes are available, then repair is not possible.
 
 ## Synopsis
 
-nix-store
-
-\--dump
-
-path
+`nix-store` `--dump` *path*
 
 ## Description
 
@@ -739,11 +626,7 @@ A Nix archive can be unpacked using `nix-store
 
 ## Synopsis
 
-nix-store
-
-\--restore
-
-path
+`nix-store` `--restore` *path*
 
 ## Description
 
@@ -754,11 +637,7 @@ not already exist. The archive is read from standard input.
 
 ## Synopsis
 
-nix-store
-
-\--export
-
-paths
+`nix-store` `--export` *paths…*
 
 ## Description
 
@@ -784,9 +663,7 @@ To import the whole closure again, run:
 
 ## Synopsis
 
-nix-store
-
-\--import
+`nix-store` `--import`
 
 ## Description
 
@@ -800,9 +677,7 @@ Nix store, the import fails.
 
 ## Synopsis
 
-nix-store
-
-\--optimise
+`nix-store` `--optimise`
 
 ## Description
 
@@ -832,13 +707,7 @@ Use `-vv` or `-vvv` to get some progress indication.
 
 ## Synopsis
 
-nix-store
-
-\--read-log
-
-\-l
-
-paths
+`nix-store` {`--read-log` | `-l`} *paths…*
 
 ## Description
 
@@ -866,11 +735,7 @@ substitute, then the log is unavailable.
 
 ## Synopsis
 
-nix-store
-
-\--dump-db
-
-paths
+`nix-store` `--dump-db` [*paths…*]
 
 ## Description
 
@@ -889,9 +754,7 @@ example.
 
 ## Synopsis
 
-nix-store
-
-\--load-db
+`nix-store` `--load-db`
 
 ## Description
 
@@ -902,11 +765,7 @@ The operation `--load-db` reads a dump of the Nix database created by
 
 ## Synopsis
 
-nix-store
-
-\--print-env
-
-drvpath
+`nix-store` `--print-env` *drvpath*
 
 ## Description
 
@@ -927,15 +786,7 @@ of the builder are placed in the variable `_args`.
 
 ## Synopsis
 
-nix-store
-
-\--generate-binary-cache-key
-
-key-name
-
-secret-key-file
-
-public-key-file
+`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file*
 
 ## Description
 
@@ -952,5 +803,3 @@ mandatory parameters:
 2.  The file name where the secret key is to be stored.
 
 3.  The file name where the public key is to be stored.
-
-# Environment variables

From 7a0e6f076a34e214832ac3a8cc6472aea5fd1559 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 12:58:31 +0200
Subject: [PATCH 041/882] Move figures

---
 doc/manual/{ => src}/figures/user-environments.png | Bin
 doc/manual/{ => src}/figures/user-environments.sxd | Bin
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename doc/manual/{ => src}/figures/user-environments.png (100%)
 rename doc/manual/{ => src}/figures/user-environments.sxd (100%)

diff --git a/doc/manual/figures/user-environments.png b/doc/manual/src/figures/user-environments.png
similarity index 100%
rename from doc/manual/figures/user-environments.png
rename to doc/manual/src/figures/user-environments.png
diff --git a/doc/manual/figures/user-environments.sxd b/doc/manual/src/figures/user-environments.sxd
similarity index 100%
rename from doc/manual/figures/user-environments.sxd
rename to doc/manual/src/figures/user-environments.sxd

From 4a79b3598f077983dd5717d7822ef8cb6a9fa967 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 13:01:00 +0200
Subject: [PATCH 042/882] Fix nix-copy-closure manpage

---
 .../src/command-ref/nix-copy-closure.md       | 46 +++++++++----------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index 4026e50de..2a4558324 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -37,37 +37,37 @@ and second to send the dump of those paths.  If this bothers you, use
 
 # Options
 
-*`--to`*
-: Copy the closure of _paths_ from the local Nix store to the Nix
-  store on _machine_. This is the default.
+  - `--to`  
+    Copy the closure of _paths_ from the local Nix store to the Nix
+    store on _machine_. This is the default.
 
-*`--from`*
-: Copy the closure of _paths_ from the Nix store on _machine_ to the
-  local Nix store.
+  - `--from`  
+    Copy the closure of _paths_ from the Nix store on _machine_ to the
+    local Nix store.
 
-*`--gzip`*
-: Enable compression of the SSH connection.
+  - `--gzip`  
+    Enable compression of the SSH connection.
 
-*`--include-outputs`*
-: Also copy the outputs of store derivations included in the closure.
+  - `--include-outputs`  
+    Also copy the outputs of store derivations included in the closure.
 
-*`--use-substitutes` / `-s`*
-: Attempt to download missing paths on the target machine using Nix’s
-  substitute mechanism.  Any paths that cannot be substituted on the
-  target are still copied normally from the source.  This is useful,
-  for instance, if the connection between the source and target
-  machine is slow, but the connection between the target machine and
-  `nixos.org` (the default binary cache server) is
-  fast.
+  - `--use-substitutes` / `-s`  
+    Attempt to download missing paths on the target machine using Nix’s
+    substitute mechanism.  Any paths that cannot be substituted on the
+    target are still copied normally from the source.  This is useful,
+    for instance, if the connection between the source and target
+    machine is slow, but the connection between the target machine and
+    `nixos.org` (the default binary cache server) is
+    fast.
 
-*`-v`*
-: Show verbose output.
+  - `-v`  
+    Show verbose output.
 
 # Environment variables
 
-*`NIX_SSHOPTS`*
-: Additional options to be passed to `ssh` on the command
-  line.
+  - `NIX_SSHOPTS`  
+    Additional options to be passed to `ssh` on the command
+    line.
 
 # Examples
 

From da3d776cb91123a4d0528251b7ce909419ca0c7a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 14:31:33 +0200
Subject: [PATCH 043/882] Fix some dangling references

---
 .../src/advanced-topics/cores-vs-jobs.md      | 35 +++++----
 doc/manual/src/advanced-topics/diff-hook.md   | 20 +++---
 .../src/advanced-topics/post-build-hook.md    |  7 +-
 doc/manual/src/command-ref/conf-file.md       | 24 ++-----
 doc/manual/src/command-ref/nix-build.md       |  2 +-
 doc/manual/src/command-ref/nix-channel.md     |  1 -
 doc/manual/src/command-ref/nix-env.md         |  2 +-
 doc/manual/src/command-ref/nix-instantiate.md |  2 -
 doc/manual/src/command-ref/nix-shell.md       |  2 +-
 doc/manual/src/command-ref/nix-store.md       | 14 ++--
 .../src/expressions/arguments-variables.md    | 22 +++---
 doc/manual/src/expressions/build-script.md    |  8 +--
 doc/manual/src/expressions/builder-syntax.md  | 72 -------------------
 doc/manual/src/expressions/builtins.md        | 69 +++++++++---------
 doc/manual/src/expressions/derivations.md     |  5 +-
 .../src/expressions/expression-syntax.md      |  7 +-
 doc/manual/src/expressions/generic-builder.md | 13 ++--
 .../src/expressions/language-operators.md     |  7 +-
 doc/manual/src/glossary.md                    | 38 +++++-----
 .../package-management/basic-package-mgmt.md  | 10 +--
 .../src/package-management/copy-closure.md    |  2 +-
 .../package-management/package-management.md  |  9 +--
 doc/manual/src/package-management/profiles.md |  3 +-
 23 files changed, 139 insertions(+), 235 deletions(-)
 delete mode 100644 doc/manual/src/expressions/builder-syntax.md

diff --git a/doc/manual/src/advanced-topics/cores-vs-jobs.md b/doc/manual/src/advanced-topics/cores-vs-jobs.md
index 91e95d5b4..4a9058ca1 100644
--- a/doc/manual/src/advanced-topics/cores-vs-jobs.md
+++ b/doc/manual/src/advanced-topics/cores-vs-jobs.md
@@ -1,33 +1,32 @@
 # Tuning Cores and Jobs
 
-Nix has two relevant settings with regards to how your CPU cores will be
-utilized: [???](#conf-cores) and [???](#conf-max-jobs). This chapter
-will talk about what they are, how they interact, and their
-configuration trade-offs.
+Nix has two relevant settings with regards to how your CPU cores will
+be utilized: `cores` and `max-jobs`. This chapter will talk about what
+they are, how they interact, and their configuration trade-offs.
 
-  - [???](#conf-max-jobs)  
+  - `max-jobs`  
     Dictates how many separate derivations will be built at the same
-    time. If you set this to zero, the local machine will do no builds.
-    Nix will still substitute from binary caches, and build remotely if
-    remote builders are configured.
+    time. If you set this to zero, the local machine will do no
+    builds.  Nix will still substitute from binary caches, and build
+    remotely if remote builders are configured.
 
-  - [???](#conf-cores)  
-    Suggests how many cores each derivation should use. Similar to `make
-    -j`.
+  - `cores`  
+    Suggests how many cores each derivation should use. Similar to
+    `make -j`.
 
-The [???](#conf-cores) setting determines the value of
-`NIX_BUILD_CORES`. `NIX_BUILD_CORES` is equal to [???](#conf-cores),
-unless [???](#conf-cores) equals `0`, in which case `NIX_BUILD_CORES`
-will be the total number of cores in the system.
+The `cores` setting determines the value of
+`NIX_BUILD_CORES`. `NIX_BUILD_CORES` is equal to `cores`, unless
+`cores` equals `0`, in which case `NIX_BUILD_CORES` will be the total
+number of cores in the system.
 
 The maximum number of consumed cores is a simple multiplication,
-[???](#conf-max-jobs) \* `NIX_BUILD_CORES`.
+`max-jobs` \* `NIX_BUILD_CORES`.
 
 The balance on how to set these two independent variables depends upon
 each builder's workload and hardware. Here are a few example scenarios
 on a machine with 24 cores:
 
-| [???](#conf-max-jobs) | [???](#conf-cores) | `NIX_BUILD_CORES` | Maximum Processes | Result                                                                                                                                                                                                                                                                                 |
+| `max-jobs` | `cores` | `NIX_BUILD_CORES` | Maximum Processes | Result                                                                                                                                                                                                                                                                                 |
 | --------------------- | ------------------ | ----------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | 1                     | 24                 | 24                | 24                | One derivation will be built at a time, each one can use 24 cores. Undersold if a job can’t use 24 cores.                                                                                                                                                                              |
 | 4                     | 6                  | 6                 | 24                | Four derivations will be built at once, each given access to six cores.                                                                                                                                                                                                                |
@@ -35,8 +34,6 @@ on a machine with 24 cores:
 | 24                    | 1                  | 1                 | 24                | 24 derivations can build at the same time, each using a single core. Never oversold, but derivations which require many cores will be very slow to compile.                                                                                                                            |
 | 24                    | 0                  | 24                | 576               | 24 derivations can build at the same time, each using all the available cores of the machine. Very likely to be oversold, and very likely to suffer context switches.                                                                                                                  |
 
-Balancing 24 Build Cores
-
 It is up to the derivations' build script to respect host's requested
 cores-per-build by following the value of the `NIX_BUILD_CORES`
 environment variable.
diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md
index 2c9896fa5..e2234147f 100644
--- a/doc/manual/src/advanced-topics/diff-hook.md
+++ b/doc/manual/src/advanced-topics/diff-hook.md
@@ -1,9 +1,8 @@
 # Verifying Build Reproducibility
 
-Specify a program with Nix's [???](#conf-diff-hook) to compare build
-results when two builds produce different results. Note: this hook is
-only executed if the results are not the same, this hook is not used for
-determining if the results are the same.
+You can use Nix's `diff-hook` setting to compare build results. Note
+that this hook is only executed if the results differ; it is not used
+for determining if the results are the same.
 
 For purposes of demonstration, we'll use the following Nix file,
 `deterministic.nix` for testing:
@@ -93,7 +92,7 @@ has copied the build results to that directory where you can examine it.
 > path will be deleted on the next garbage collection.
 > 
 > The path is guaranteed to be alive for the duration of
-> [???](#conf-diff-hook)'s execution, but may be deleted any time after.
+> the `diff-hook`'s execution, but may be deleted any time after.
 > 
 > If the comparison is performed as part of automated tooling, please
 > use the diff-hook or author your tooling to handle the case where the
@@ -112,9 +111,8 @@ Run the build without `--check`, and then try with `--check` again.
 Automatically verify every build at build time by executing the build
 multiple times.
 
-Setting [???](#conf-repeat) and [???](#conf-enforce-determinism) in your
-`nix.conf` permits the automated verification of every build Nix
-performs.
+Setting `repeat` and `enforce-determinism` in your `nix.conf` permits
+the automated verification of every build Nix performs.
 
 The following configuration will run each build three times, and will
 require the build to be deterministic:
@@ -122,9 +120,9 @@ require the build to be deterministic:
     enforce-determinism = true
     repeat = 2
 
-Setting [???](#conf-enforce-determinism) to false as in the following
-configuration will run the build multiple times, execute the build hook,
-but will allow the build to succeed even if it does not build
+Setting `enforce-determinism` to false as in the following
+configuration will run the build multiple times, execute the build
+hook, but will allow the build to succeed even if it does not build
 reproducibly:
 
     enforce-determinism = false
diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md
index 166b57da6..0d02a8ff3 100644
--- a/doc/manual/src/advanced-topics/post-build-hook.md
+++ b/doc/manual/src/advanced-topics/post-build-hook.md
@@ -17,9 +17,8 @@ the build loop.
 
 # Prerequisites
 
-This tutorial assumes you have configured an S3-compatible binary cache
-according to the instructions at
-[???](#ssec-s3-substituter-authenticated-writes), and that the `root`
+This tutorial assumes you have [configured an S3-compatible binary
+cache](../package-management/s3-substituter.md), and that the `root`
 user's default AWS profile can upload to the bucket.
 
 # Set up a Signing Key
@@ -33,7 +32,7 @@ distribute the public key for verifying the authenticity of the paths.
     example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
 
 Then, add the public key and the cache URL to your `nix.conf`'s
-[???](#conf-trusted-public-keys) and [???](#conf-substituters) like:
+`trusted-public-keys` and `substituters` options:
 
     substituters = https://cache.nixos.org/ s3://example-nix-cache
     trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
index 4fa9cedfc..bedf1ec6b 100644
--- a/doc/manual/src/command-ref/conf-file.md
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -77,8 +77,7 @@ The following settings are currently available:
                     --optimise` to get rid of duplicate files.
 
   - `builders`  
-    A list of machines on which to perform builds. See
-    [???](#chap-distributed-builds) for details.
+    A list of machines on which to perform builds.
 
   - `builders-use-substitutes`  
     If set to `true`, Nix will instruct remote build machines to use
@@ -140,8 +139,6 @@ The following settings are currently available:
     `-jN` flag to GNU Make. It can be overridden using the `--cores`
     command line switch and defaults to `1`. The value `0` means that
     the builder should use all available CPU cores in the system.
-    
-    See also [???](#chap-tuning-cores-and-jobs).
 
   - `diff-hook`  
     Absolute path to an executable capable of diffing build results. The
@@ -298,8 +295,6 @@ The following settings are currently available:
     `preferLocalBuild` derivation attribute which executes locally
     regardless). It can be overridden using the `--max-jobs` (`-j`)
     command line switch.
-    
-    See also [???](#chap-tuning-cores-and-jobs).
 
   - `max-silent-time`  
     This option defines the maximum number of seconds that a builder can
@@ -429,12 +424,10 @@ The following settings are currently available:
         
         Example:
         `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
-                                                                                                        /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
-                                                                                                        /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
-                                                                                                        /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
-                                                                                                        /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
-    
-    See [???](#chap-post-build-hook) for an example implementation.
+        /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+        /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+        /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+        /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
 
   - `repeat`  
     How many times to repeat builds to check whether they are
@@ -459,8 +452,7 @@ The following settings are currently available:
     `allowed-uri`. The default is `false`.
 
   - `run-diff-hook`  
-    If true, enable the execution of
-    [varlistentry\_title](#conf-diff-hook).
+    If true, enable the execution of the `diff-hook` program.
     
     When using the Nix daemon, `run-diff-hook` must be set in the
     `nix.conf` configuration file, and cannot be passed at the command
@@ -595,15 +587,11 @@ The following settings are currently available:
     Nix will print a log message at the "vomit" level for every function
     entrance and function exit.
     
-    <div class="informalexample">
-    
         function-trace entered undefined position at 1565795816999559622
         function-trace exited undefined position at 1565795816999581277
         function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
         function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
     
-    </div>
-    
     The `undefined position` means the function call is a builtin.
     
     Use the `contrib/stack-collapse.py` script distributed with the Nix
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index a65f53b4b..81dc26a39 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -46,7 +46,7 @@ expression to a low-level store derivation) and [`nix-store
 
 All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
-`nix-instantiate`. See also [???](#sec-common-options).
+`nix-instantiate`.
 
   - `--no-out-link`  
     Do not create a symlink to the output path. Note that as a result
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index 6b4bd2459..ea3a57e69 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -13,7 +13,6 @@ Title: nix-channel
 A Nix channel is a mechanism that allows you to automatically stay
 up-to-date with a set of pre-built Nix expressions. A Nix channel is
 just a URL that points to a place containing a set of Nix expressions.
-See also [???](#sec-channels).
 
 To see the list of official NixOS channels, visit
 <https://nixos.org/channels>.
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index cf688100e..ab1d10c08 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -66,7 +66,7 @@ match. Here are some examples:
 
 This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
-have an effect. See also [???](#sec-common-options).
+have an effect.
 
   - `--file` / `-f` *path*  
     Specifies the Nix expression (designated below as the *active Nix
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index b6bbbe80a..ca9ef1fc9 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -30,8 +30,6 @@ the resulting store derivations are printed on standard output.
 If *files* is the character `-`, then a Nix expression will be read from
 standard input.
 
-See also [???](#sec-common-options) for a list of common options.
-
 # Options
 
   - `--add-root` *path*; `--indirect`  
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index d6dbb6e26..492351867 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -50,7 +50,7 @@ will cause `nix-shell` to print `Hello shell`.
 
 All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
-`nix-instantiate`. See also [???](#sec-common-options).
+`nix-instantiate`.
 
   - `--command` *cmd*  
     In the environment of the derivation, run the shell command *cmd*.
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index a098beb1e..b15340fde 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -23,16 +23,15 @@ subcommand to be performed. These are documented below.
 
 This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
-have an effect. See also [???](#sec-common-options) for a list of common
-options.
+have an effect.
 
   - `--add-root` *path*  
     Causes the result of a realisation (`--realise` and
     `--force-realise`) to be registered as a root of the garbage
-    collector(see [???](#ssec-gc-roots)). The root is stored in *path*,
-    which must be inside a directory that is scanned for roots by the
-    garbage collector (i.e., typically in a subdirectory of
-    `/nix/var/nix/gcroots/`) *unless* the `--indirect` flag is used.
+    collector. The root is stored in *path*, which must be inside a
+    directory that is scanned for roots by the garbage collector
+    (i.e., typically in a subdirectory of `/nix/var/nix/gcroots/`)
+    *unless* the `--indirect` flag is used.
 
     If there are multiple results, then multiple symlinks will be
     created by sequentially numbering symlinks beyond the first one
@@ -209,8 +208,7 @@ The following suboperations may be specified:
 
   - `--print-roots`  
     This operation prints on standard output the set of roots used by
-    the garbage collector. What constitutes a root is described in
-    [???](#ssec-gc-roots).
+    the garbage collector.
 
   - `--print-live`  
     This operation prints on standard output the set of “live” store
diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md
index 436ae559d..2956f98f1 100644
--- a/doc/manual/src/expressions/arguments-variables.md
+++ b/doc/manual/src/expressions/arguments-variables.md
@@ -1,11 +1,12 @@
 # Arguments and Variables
 
-The Nix expression in [???](#ex-hello-nix) is a function; it is missing
-some arguments that have to be filled in somewhere. In the Nix Packages
-collection this is done in the file `pkgs/top-level/all-packages.nix`,
-where all Nix expressions for packages are imported and called with the
-appropriate arguments. Here are some fragments of `all-packages.nix`,
-with annotations of what they mean:
+The [Nix expression for GNU Hello](expression-syntax.md) is a
+function; it is missing some arguments that have to be filled in
+somewhere. In the Nix Packages collection this is done in the file
+`pkgs/top-level/all-packages.nix`, where all Nix expressions for
+packages are imported and called with the appropriate arguments. Here
+are some fragments of `all-packages.nix`, with annotations of what
+they mean:
 
     ...
     
@@ -35,9 +36,10 @@ with annotations of what they mean:
 
 2.  Here we *import* the Nix expression for GNU Hello. The import
     operation just loads and returns the specified Nix expression. In
-    fact, we could just have put the contents of [???](#ex-hello-nix) in
-    `all-packages.nix` at this point. That would be completely
-    equivalent, but it would make the file rather bulky.
+    fact, we could just have put the contents of the Nix expression
+    for GNU Hello in `all-packages.nix` at this point. That would be
+    completely equivalent, but it would make `all-packages.nix` rather
+    bulky.
     
     Note that we refer to `../applications/misc/hello/ex-1`, not
     `../applications/misc/hello/ex-1/default.nix`. When you try to
@@ -54,7 +56,7 @@ with annotations of what they mean:
     The result of this function call is an actual derivation that can be
     built by Nix (since when we fill in the arguments of the function,
     what we get is its body, which is the call to `stdenv.mkDerivation`
-    in [???](#ex-hello-nix)).
+    in the [Nix expression for GNU Hello](expression-syntax.md)).
     
     > **Note**
     > 
diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md
index 57222de85..e0dda56f8 100644
--- a/doc/manual/src/expressions/build-script.md
+++ b/doc/manual/src/expressions/build-script.md
@@ -25,10 +25,10 @@ steps to elucidate what a builder does. It performs the following steps:
     
     So the first step is to set up the environment. This is done by
     calling the `setup` script of the standard environment. The
-    environment variable `stdenv` points to the location of the standard
-    environment being used. (It wasn't specified explicitly as an
-    attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
-    automatically.)
+    environment variable `stdenv` points to the location of the
+    standard environment being used. (It wasn't specified explicitly
+    as an attribute in Hello's Nix expression, but `mkDerivation` adds
+    it automatically.)
 
 2.  Since Hello needs Perl, we have to make sure that Perl is in the
     `PATH`. The `perl` environment variable points to the location of
diff --git a/doc/manual/src/expressions/builder-syntax.md b/doc/manual/src/expressions/builder-syntax.md
deleted file mode 100644
index 916159d40..000000000
--- a/doc/manual/src/expressions/builder-syntax.md
+++ /dev/null
@@ -1,72 +0,0 @@
-# Builder Syntax
-
-    source $stdenv/setup 
-    
-    PATH=$perl/bin:$PATH 
-    
-    tar xvfz $src 
-    cd hello-*
-    ./configure --prefix=$out 
-    make 
-    make install
-
-[example\_title](#ex-hello-builder) shows the builder referenced from
-Hello's Nix expression (stored in
-`pkgs/applications/misc/hello/ex-1/builder.sh`). The builder can
-actually be made a lot shorter by using the *generic builder* functions
-provided by `stdenv`, but here we write out the build steps to elucidate
-what a builder does. It performs the following steps:
-
-  - When Nix runs a builder, it initially completely clears the
-    environment (except for the attributes declared in the derivation).
-    For instance, the `PATH` variable is empty\[1\]. This is done to
-    prevent undeclared inputs from being used in the build process. If
-    for example the `PATH` contained `/usr/bin`, then you might
-    accidentally use `/usr/bin/gcc`.
-    
-    So the first step is to set up the environment. This is done by
-    calling the `setup` script of the standard environment. The
-    environment variable `stdenv` points to the location of the standard
-    environment being used. (It wasn't specified explicitly as an
-    attribute in [???](#ex-hello-nix), but `mkDerivation` adds it
-    automatically.)
-
-  - Since Hello needs Perl, we have to make sure that Perl is in the
-    `PATH`. The `perl` environment variable points to the location of
-    the Perl package (since it was passed in as an attribute to the
-    derivation), so `$perl/bin` is the directory containing the Perl
-    interpreter.
-
-  - Now we have to unpack the sources. The `src` attribute was bound to
-    the result of fetching the Hello source tarball from the network, so
-    the `src` environment variable points to the location in the Nix
-    store to which the tarball was downloaded. After unpacking, we `cd`
-    to the resulting source directory.
-    
-    The whole build is performed in a temporary directory created in
-    `/tmp`, by the way. This directory is removed after the builder
-    finishes, so there is no need to clean up the sources afterwards.
-    Also, the temporary directory is always newly created, so you don't
-    have to worry about files from previous builds interfering with the
-    current build.
-
-  - GNU Hello is a typical Autoconf-based package, so we first have to
-    run its `configure` script. In Nix every package is stored in a
-    separate location in the Nix store, for instance
-    `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix
-    computes this path by cryptographically hashing all attributes of
-    the derivation. The path is passed to the builder through the `out`
-    environment variable. So here we give `configure` the parameter
-    `--prefix=$out` to cause Hello to be installed in the expected
-    location.
-
-  - Finally we build Hello (`make`) and install it into the location
-    specified by `out` (`make install`).
-
-If you are wondering about the absence of error checking on the result
-of various commands called in the builder: this is because the shell
-script is evaluated with Bash's `-e` option, which causes the script to
-be aborted if any command fails without an error check.
-
-1.  Actually, it's initialised to `/path-not-set` to prevent Bash from
-    setting it to a default value.
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index 8f19c692a..3c9bb4533 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -83,7 +83,7 @@ For instance, `derivation` is also available as `builtins.derivation`.
     its elements or attributes are also evaluated recursively.
 
   - `derivation` *attrs*; `builtins.derivation` *attrs*  
-    `derivation` is described in [???](#ssec-derivation).
+    `derivation` is described in [its own section](derivations.md).
 
   - `dirOf` *s*; `builtins.dirOf` *s*  
     Return the directory part of the string *s*, that is, everything
@@ -233,8 +233,8 @@ For instance, `derivation` is also available as `builtins.derivation`.
         
         > **Note**
         > 
-        > Nix will refetch the branch in accordance to
-        > [???](#conf-tarball-ttl).
+        > Nix will refetch the branch in accordance with
+        > the option `tarball-ttl`.
         
         > **Note**
         > 
@@ -351,19 +351,18 @@ For instance, `derivation` is also available as `builtins.derivation`.
 
   - `import` *path*; `builtins.import` *path*  
     Load, parse and return the Nix expression in the file *path*. If
-    *path* is a directory, the file ` default.nix
-                     ` in that directory is loaded. Evaluation aborts if the file
-    doesn’t exist or contains an incorrect Nix expression. `import`
-    implements Nix’s module system: you can put any Nix expression (such
-    as a set or a function) in a separate file, and use it from Nix
-    expressions in other files.
+    *path* is a directory, the file ` default.nix ` in that directory
+    is loaded. Evaluation aborts if the file doesn’t exist or contains
+    an incorrect Nix expression. `import` implements Nix’s module
+    system: you can put any Nix expression (such as a set or a
+    function) in a separate file, and use it from Nix expressions in
+    other files.
     
     > **Note**
     > 
     > Unlike some languages, `import` is a regular function in Nix.
-    > Paths using the angle bracket syntax (e.g., `
-    >     >     >     >     > import` *\<foo\>*) are normal path values (see
-    > [???](#ssec-values)).
+    > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*)
+    > are [normal path values](language-values.md).
     
     A Nix expression loaded by `import` must not contain any *free
     variables* (identifiers that are not defined in the Nix expression
@@ -643,11 +642,12 @@ For instance, `derivation` is also available as `builtins.derivation`.
     (which is not the case for `abort`).
 
   - `builtins.toFile` *name* *s*  
-    Store the string *s* in a file in the Nix store and return its path.
-    The file has suffix *name*. This file can be used as an input to
-    derivations. One application is to write builders “inline”. For
-    instance, the following Nix expression combines [???](#ex-hello-nix)
-    and [???](#ex-hello-builder) into one file:
+    Store the string *s* in a file in the Nix store and return its
+    path.  The file has suffix *name*. This file can be used as an
+    input to derivations. One application is to write builders
+    “inline”. For instance, the following Nix expression combines the
+    [Nix expression for GNU Hello](expression-syntax.md) and its
+    [build script](build-script.md) into one file:
     
         { stdenv, fetchurl, perl }:
         
@@ -688,8 +688,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
       ";
     ```
     
-    Note that `${configFile}` is an antiquotation (see
-    [???](#ssec-values)), so the result of the expression `configFile`
+    Note that `${configFile}` is an
+    [antiquotation](language-values.md), so the result of the
+    expression `configFile`
     (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
     spliced into the resulting string.
     
@@ -786,17 +787,17 @@ For instance, `derivation` is also available as `builtins.derivation`.
     container contains a number of servlets (`*.war` files) each
     exported under a specific URI prefix. So the servlet configuration
     is a list of sets containing the `path` and `war` of the servlet
-    ([???](#ex-toxml-co-servlets)). This kind of information is
-    difficult to communicate with the normal method of passing
-    information through an environment variable, which just concatenates
-    everything together into a string (which might just work in this
-    case, but wouldn’t work if fields are optional or contain lists
-    themselves). Instead the Nix expression is converted to an XML
-    representation with `toXML`, which is unambiguous and can easily be
-    processed with the appropriate tools. For instance, in the example
-    an XSLT stylesheet (at point ②) is applied to it (at point ①) to
-    generate the XML configuration file for the Jetty server. The XML
-    representation produced at point ③ by `toXML` is as follows:
+    (①). This kind of information is difficult to communicate with the
+    normal method of passing information through an environment
+    variable, which just concatenates everything together into a
+    string (which might just work in this case, but wouldn’t work if
+    fields are optional or contain lists themselves). Instead the Nix
+    expression is converted to an XML representation with `toXML`,
+    which is unambiguous and can easily be processed with the
+    appropriate tools. For instance, in the example an XSLT stylesheet
+    (at point ②) is applied to it (at point ①) to generate the XML
+    configuration file for the Jetty server. The XML representation
+    produced at point ③ by `toXML` is as follows:
     
         <?xml version='1.0' encoding='utf-8'?>
         <expr>
@@ -820,10 +821,10 @@ For instance, `derivation` is also available as `builtins.derivation`.
           </list>
         </expr>
     
-    Note that [???](#ex-toxml) uses the `toFile` built-in to write the
-    builder and the stylesheet “inline” in the Nix expression. The path
-    of the stylesheet is spliced into the builder using the syntax
-    `xsltproc ${stylesheet}`.
+    Note that we used the `toFile` built-in to write the builder and
+    the stylesheet “inline” in the Nix expression. The path of the
+    stylesheet is spliced into the builder using the syntax `xsltproc
+    ${stylesheet}`.
 
   - `builtins.trace` *e1* *e2*  
     Evaluate *e1* and print its abstract syntax representation on
diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/expressions/derivations.md
index 4cb233e2e..0e5656b5b 100644
--- a/doc/manual/src/expressions/derivations.md
+++ b/doc/manual/src/expressions/derivations.md
@@ -9,8 +9,9 @@ the attributes of which specify the inputs of the build.
     `"x86_64-darwin"`. (To figure out your system type, run `nix -vv
     --version`.) The build can only be performed on a machine and
     operating system matching the system type. (Nix can automatically
-    forward builds for other platforms by forwarding them to other
-    machines; see [???](#chap-distributed-builds).)
+    [forward builds for other
+    platforms](../advanced-topics/distributed-builds.md) by forwarding
+    them to other machines.)
 
   - There must be an attribute named `name` whose value must be a
     string. This is used as a symbolic name for the package by
diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md
index 2abe6ac6e..e3432b577 100644
--- a/doc/manual/src/expressions/expression-syntax.md
+++ b/doc/manual/src/expressions/expression-syntax.md
@@ -61,9 +61,10 @@ elements (referenced from the figure by number):
     sometimes be omitted, in which case `mkDerivation` will fill in a
     default builder (which does a `configure; make; make install`, in
     essence). Hello is sufficiently simple that the default builder
-    would suffice, but in this case, we will show an actual builder for
-    educational purposes. The value `./builder.sh` refers to the shell
-    script shown in [???](#ex-hello-builder), discussed below.
+    would suffice, but in this case, we will show an actual builder
+    for educational purposes. The value `./builder.sh` refers to the
+    shell script shown in the [next section](build-script.md),
+    discussed below.
 
 5.  The builder has to know what the sources of the package are. Here,
     the attribute `src` is bound to the result of a call to the
diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md
index 6942abe70..43275dbf7 100644
--- a/doc/manual/src/expressions/generic-builder.md
+++ b/doc/manual/src/expressions/generic-builder.md
@@ -1,7 +1,7 @@
 # Generic Builder Syntax
 
-Recall from [???](#ex-hello-builder) that the builder looked something
-like this:
+Recall that the [build script for GNU Hello](build-script.md) looked
+something like this:
 
     PATH=$perl/bin:$PATH
     tar xvfz $src
@@ -37,11 +37,10 @@ Here is what each line means:
 2.  The function `genericBuild` is defined in the file `$stdenv/setup`.
 
 3.  The final step calls the shell function `genericBuild`, which
-    performs the steps that were done explicitly in
-    [???](#ex-hello-builder). The generic builder is smart enough to
-    figure out whether to unpack the sources using `gzip`, `bzip2`, etc.
-    It can be customised in many ways; see the Nixpkgs manual for
-    details.
+    performs the steps that were done explicitly in the previous build
+    script. The generic builder is smart enough to figure out whether
+    to unpack the sources using `gzip`, `bzip2`, etc.  It can be
+    customised in many ways; see the Nixpkgs manual for details.
 
 Discerning readers will note that the `buildInputs` could just as well
 have been set in the Nix expression, like this:
diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md
index b124a2417..1d787ffe3 100644
--- a/doc/manual/src/expressions/language-operators.md
+++ b/doc/manual/src/expressions/language-operators.md
@@ -1,8 +1,7 @@
 # Operators
 
-[table\_title](#table-operators) lists the operators in the Nix
-expression language, in order of precedence (from strongest to weakest
-binding).
+The table below lists the operators in the Nix expression language, in
+order of precedence (from strongest to weakest binding).
 
 | Name                     | Syntax                              | Associativity | Description                                                                                                                                                                                                                   | Precedence |
 | ------------------------ | ----------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
@@ -28,5 +27,3 @@ binding).
 | Logical OR               | *e1* `\|\|` *e2*                    | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
 | Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to `!e1 \|\|
         e2`).                                                                                                                                                                    | 14         |
-
-Operators
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index c8bdf4fd2..56af9cd85 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -3,7 +3,7 @@
   - derivation  
     A description of a build action. The result of a derivation is a
     store object. Derivations are typically specified in Nix expressions
-    using the [`derivation` primitive](#ssec-derivation). These are
+    using the [`derivation` primitive](expressions/derivations.md). These are
     translated into low-level *store derivations* (implicitly by
     `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
@@ -53,20 +53,19 @@
     paths.
 
   - reachable  
-    A store path `Q` is reachable from another store path `P` if `Q` is
-    in the [closure](#gloss-closure) of the
-    [references](#gloss-reference) relation.
+    A store path `Q` is reachable from another store path `P` if `Q`
+    is in the *closure* of the *references* relation.
 
   - closure  
     The closure of a store path is the set of store paths that are
     directly or indirectly “reachable” from that store path; that is,
-    it’s the closure of the path under the
-    [references](#gloss-reference) relation. For a package, the closure
-    of its derivation is equivalent to the build-time dependencies,
-    while the closure of its output path is equivalent to its runtime
-    dependencies. For correct deployment it is necessary to deploy whole
-    closures, since otherwise at runtime files could be missing. The
-    command `nix-store -qR` prints out closures of store paths.
+    it’s the closure of the path under the *references* relation. For
+    a package, the closure of its derivation is equivalent to the
+    build-time dependencies, while the closure of its output path is
+    equivalent to its runtime dependencies. For correct deployment it
+    is necessary to deploy whole closures, since otherwise at runtime
+    files could be missing. The command `nix-store -qR` prints out
+    closures of store paths.
     
     As an example, if the store object at path `P` contains a reference
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
@@ -76,7 +75,7 @@
     A store path produced by a derivation.
 
   - deriver  
-    The deriver of an [output path](#gloss-output-path) is the store
+    The deriver of an *output path* is the store
     derivation that built it.
 
   - validity  
@@ -87,16 +86,15 @@
   - user environment  
     An automatically generated store object that consists of a set of
     symlinks to “active” applications, i.e., other store paths. These
-    are generated automatically by [`nix-env`](#sec-nix-env). See
-    [???](#sec-profiles).
+    are generated automatically by
+    [`nix-env`](command-ref/nix-env.md). See *profiles*.
 
   - profile  
-    A symlink to the current [user environment](#gloss-user-env) of a
-    user, e.g., `/nix/var/nix/profiles/default`.
+    A symlink to the current *user environment* of a user, e.g.,
+    `/nix/var/nix/profiles/default`.
 
   - NAR  
     A *N*ix *AR*chive. This is a serialisation of a path in the Nix
-    store. It can contain regular files, directories and symbolic links.
-    NARs are generated and unpacked using `nix-store --dump` and
-    `nix-store
-            --restore`.
+    store. It can contain regular files, directories and symbolic
+    links.  NARs are generated and unpacked using `nix-store --dump`
+    and `nix-store --restore`.
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
index d9c34afc6..8edaca6b0 100644
--- a/doc/manual/src/package-management/basic-package-mgmt.md
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -24,11 +24,11 @@ or completely new ones.)
 
 You can manually download the latest version of Nixpkgs from
 <http://nixos.org/nixpkgs/download.html>. However, it’s much more
-convenient to use the Nixpkgs *channel*, since it makes it easy to stay
-up to date with new versions of Nixpkgs. (Channels are described in more
-detail in [???](#sec-channels).) Nixpkgs is automatically added to your
-list of “subscribed” channels when you install Nix. If this is not the
-case for some reason, you can add it as follows:
+convenient to use the Nixpkgs [*channel*](channels.md), since it makes
+it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
+automatically added to your list of “subscribed” channels when you
+install Nix. If this is not the case for some reason, you can add it
+as follows:
 
     $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
     $ nix-channel --update
diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md
index d78b77e36..d3fac4d76 100644
--- a/doc/manual/src/package-management/copy-closure.md
+++ b/doc/manual/src/package-management/copy-closure.md
@@ -8,7 +8,7 @@ dependencies:
 
     $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)
 
-See [???](#sec-nix-copy-closure) for details.
+See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details.
 
 With `nix-store
 --export` and `nix-store --import` you can write the closure of a store
diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/package-management.md
index a2d078c16..bd26a09ab 100644
--- a/doc/manual/src/package-management/package-management.md
+++ b/doc/manual/src/package-management/package-management.md
@@ -1,4 +1,5 @@
-This chapter discusses how to do package management with Nix, i.e., how
-to obtain, install, upgrade, and erase packages. This is the “user’s”
-perspective of the Nix system — people who want to *create* packages
-should consult [???](#chap-writing-nix-expressions).
+This chapter discusses how to do package management with Nix, i.e.,
+how to obtain, install, upgrade, and erase packages. This is the
+“user’s” perspective of the Nix system — people who want to *create*
+packages should consult the [chapter on writing Nix
+expressions](../expressions/writing-nix-expressions.md).
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
index 1950c3121..f3786072d 100644
--- a/doc/manual/src/package-management/profiles.md
+++ b/doc/manual/src/package-management/profiles.md
@@ -104,8 +104,7 @@ These commands switch to the `my-profile` and default profile,
 respectively. If the profile doesn’t exist, it will be created
 automatically. You should be careful about storing a profile in another
 location than the `profiles` directory, since otherwise it might not be
-used as a root of the garbage collector (see
-[???](#sec-garbage-collection)).
+used as a root of the [garbage collector](garbage-collection.md).
 
 All `nix-env` operations work on the profile pointed to by
 `~/.nix-profile`, but you can override this using the `--profile` option

From 05a282295f3d454c811f9bdd9b755f6a5c07c190 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 15:46:16 +0200
Subject: [PATCH 044/882] Fix internal links

---
 .../src/advanced-topics/distributed-builds.md |  3 +-
 .../src/advanced-topics/post-build-hook.md    |  3 +-
 doc/manual/src/command-ref/conf-file.md       | 99 ++++++++-----------
 doc/manual/src/command-ref/env-common.md      |  7 +-
 doc/manual/src/command-ref/nix-build.md       |  5 +-
 .../src/command-ref/nix-collect-garbage.md    | 20 ++--
 doc/manual/src/command-ref/nix-env.md         | 37 +++----
 doc/manual/src/command-ref/nix-hash.md        | 14 +--
 doc/manual/src/command-ref/nix-instantiate.md | 10 +-
 doc/manual/src/command-ref/nix-store.md       | 50 +++++-----
 doc/manual/src/command-ref/opt-common.md      | 84 ++++++++--------
 .../src/expressions/advanced-attributes.md    | 30 +++---
 doc/manual/src/expressions/builtins.md        | 39 ++++----
 .../expressions/simple-building-testing.md    | 10 +-
 .../src/installation/installing-binary.md     |  4 +-
 doc/manual/src/installation/multi-user.md     |  8 +-
 .../package-management/basic-package-mgmt.md  |  7 +-
 doc/manual/src/package-management/channels.md |  4 +-
 .../src/package-management/s3-substituter.md  | 10 +-
 doc/manual/src/release-notes/rl-2.0.md        | 41 ++++----
 doc/manual/src/release-notes/rl-2.1.md        | 22 ++---
 21 files changed, 245 insertions(+), 262 deletions(-)

diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
index 4f24febb0..76a5380bf 100644
--- a/doc/manual/src/advanced-topics/distributed-builds.md
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -134,8 +134,7 @@ causes the list of machines in `/etc/nix/machines` to be included. (This
 is the default.)
 
 If you want the builders to use caches, you likely want to set the
-option [`builders-use-substitutes`](#conf-builders-use-substitutes) in
-your local `nix.conf`.
+option `builders-use-substitutes` in your local `nix.conf`.
 
 To build only on remote builders and disable building on the local
 machine, you can use the option `--max-jobs 0`.
diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md
index 0d02a8ff3..7b3ae58fb 100644
--- a/doc/manual/src/advanced-topics/post-build-hook.md
+++ b/doc/manual/src/advanced-topics/post-build-hook.md
@@ -108,5 +108,4 @@ We now have a Nix installation configured to automatically sign and
 upload every local build to a remote binary cache.
 
 Before deploying this to production, be sure to consider the
-implementation caveats in [Implementation
-Caveats](#chap-post-build-hook-caveats).
+[implementation caveats](#implementation-caveats).
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
index bedf1ec6b..306ad23f5 100644
--- a/doc/manual/src/command-ref/conf-file.md
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -141,10 +141,10 @@ The following settings are currently available:
     the builder should use all available CPU cores in the system.
 
   - `diff-hook`  
-    Absolute path to an executable capable of diffing build results. The
-    hook executes if [varlistentry\_title](#conf-run-diff-hook) is true,
-    and the output of a build is known to not be the same. This program
-    is not executed to determine if two results are the same.
+    Absolute path to an executable capable of diffing build
+    results. The hook is executed if `run-diff-hook` is true, and the
+    output of a build is known to not be the same. This program is not
+    executed to determine if two results are the same.
     
     The diff hook is executed by the same user and group who ran the
     build. However, the diff hook does not have write access to the
@@ -169,7 +169,7 @@ The following settings are currently available:
     configuration file, and cannot be passed at the command line.
 
   - `enforce-determinism`  
-    See [varlistentry\_title](#conf-repeat).
+    See `repeat`.
 
   - `extra-sandbox-paths`  
     A list of additional paths appended to `sandbox-paths`. Useful if
@@ -343,7 +343,7 @@ The following settings are currently available:
         password my-password
     
     For the exact syntax, see [the `curl`
-    documentation.](https://ec.haxx.se/usingcurl-netrc.html)
+    documentation](https://ec.haxx.se/usingcurl-netrc.html).
     
     > **Note**
     > 
@@ -434,9 +434,9 @@ The following settings are currently available:
     deterministic. The default value is 0. If the value is non-zero,
     every build is repeated the specified number of times. If the
     contents of any of the runs differs from the previous ones and
-    [varlistentry\_title](#conf-enforce-determinism) is true, the build
-    is rejected and the resulting store paths are not registered as
-    “valid” in Nix’s database.
+    `enforce-determinism` is true, the build is rejected and the
+    resulting store paths are not registered as “valid” in Nix’s
+    database.
 
   - `require-sigs`  
     If set to `true` (the default), any non-content-addressed path added
@@ -461,20 +461,21 @@ The following settings are currently available:
   - `sandbox`  
     If set to `true`, builds will be performed in a *sandboxed
     environment*, i.e., they’re isolated from the normal file system
-    hierarchy and will only see their dependencies in the Nix store, the
-    temporary build directory, private versions of `/proc`, `/dev`,
-    `/dev/shm` and `/dev/pts` (on Linux), and the paths configured with
-    the [`sandbox-paths` option](#conf-sandbox-paths). This is useful to
+    hierarchy and will only see their dependencies in the Nix store,
+    the temporary build directory, private versions of `/proc`,
+    `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths
+    configured with the `sandbox-paths` option. This is useful to
     prevent undeclared dependencies on files in directories such as
-    `/usr/bin`. In addition, on Linux, builds run in private PID, mount,
-    network, IPC and UTS namespaces to isolate them from other processes
-    in the system (except that fixed-output derivations do not run in
-    private network namespace to ensure they can access the network).
+    `/usr/bin`. In addition, on Linux, builds run in private PID,
+    mount, network, IPC and UTS namespaces to isolate them from other
+    processes in the system (except that fixed-output derivations do
+    not run in private network namespace to ensure they can access the
+    network).
     
     Currently, sandboxing only work on Linux and macOS. The use of a
     sandbox requires that Nix is run as root (so you should use the
-    [“build users” feature](#conf-build-users-group) to perform the
-    actual builds under different users than root).
+    “build users” feature to perform the actual builds under different
+    users than root).
     
     If this option is set to `relaxed`, then fixed-output derivations
     and derivations that have the `__noChroot` attribute set to `true`
@@ -631,81 +632,61 @@ The following settings are currently available:
 ## Deprecated Settings
 
   - `binary-caches`  
-    *Deprecated:* `binary-caches` is now an alias to
-    [varlistentry\_title](#conf-substituters).
+    *Deprecated:* `binary-caches` is now an alias to `substituters`.
 
   - `binary-cache-public-keys`  
-    *Deprecated:* `binary-cache-public-keys` is now an alias to
-    [varlistentry\_title](#conf-trusted-public-keys).
+    *Deprecated:* `binary-cache-public-keys` is now an alias `trusted-public-keys`.
 
   - `build-compress-log`  
-    *Deprecated:* `build-compress-log` is now an alias to
-    [varlistentry\_title](#conf-compress-build-log).
+    *Deprecated:* `build-compress-log` is now an alias to `compress-build-log`.
 
   - `build-cores`  
-    *Deprecated:* `build-cores` is now an alias to
-    [varlistentry\_title](#conf-cores).
+    *Deprecated:* `build-cores` is now an alias to `cores`.
 
   - `build-extra-chroot-dirs`  
-    *Deprecated:* `build-extra-chroot-dirs` is now an alias to
-    [varlistentry\_title](#conf-extra-sandbox-paths).
+    *Deprecated:* `build-extra-chroot-dirs` is now an alias to `extra-sandbox-paths`.
 
   - `build-extra-sandbox-paths`  
-    *Deprecated:* `build-extra-sandbox-paths` is now an alias to
-    [varlistentry\_title](#conf-extra-sandbox-paths).
+    *Deprecated:* `build-extra-sandbox-paths` is now an alias to `extra-sandbox-paths`.
 
   - `build-fallback`  
-    *Deprecated:* `build-fallback` is now an alias to
-    [varlistentry\_title](#conf-fallback).
+    *Deprecated:* `build-fallback` is now an alias to `fallback`.
 
   - `build-max-jobs`  
-    *Deprecated:* `build-max-jobs` is now an alias to
-    [varlistentry\_title](#conf-max-jobs).
+    *Deprecated:* `build-max-jobs` is now an alias to `max-jobs`.
 
   - `build-max-log-size`  
-    *Deprecated:* `build-max-log-size` is now an alias to
-    [varlistentry\_title](#conf-max-build-log-size).
+    *Deprecated:* `build-max-log-size` is now an alias to `max-build-log-size`.
 
   - `build-max-silent-time`  
-    *Deprecated:* `build-max-silent-time` is now an alias to
-    [varlistentry\_title](#conf-max-silent-time).
+    *Deprecated:* `build-max-silent-time` is now an alias to `max-silent-time`.
 
   - `build-repeat`  
-    *Deprecated:* `build-repeat` is now an alias to
-    [varlistentry\_title](#conf-repeat).
+    *Deprecated:* `build-repeat` is now an alias to `repeat`.
 
   - `build-timeout`  
-    *Deprecated:* `build-timeout` is now an alias to
-    [varlistentry\_title](#conf-timeout).
+    *Deprecated:* `build-timeout` is now an alias to `timeout`.
 
   - `build-use-chroot`  
-    *Deprecated:* `build-use-chroot` is now an alias to
-    [varlistentry\_title](#conf-sandbox).
+    *Deprecated:* `build-use-chroot` is now an alias to `sandbox`.
 
   - `build-use-sandbox`  
-    *Deprecated:* `build-use-sandbox` is now an alias to
-    [varlistentry\_title](#conf-sandbox).
+    *Deprecated:* `build-use-sandbox` is now an alias to `sandbox`.
 
   - `build-use-substitutes`  
-    *Deprecated:* `build-use-substitutes` is now an alias to
-    [varlistentry\_title](#conf-substitute).
+    *Deprecated:* `build-use-substitutes` is now an alias to `substitute`.
 
   - `gc-keep-derivations`  
-    *Deprecated:* `gc-keep-derivations` is now an alias to
-    [varlistentry\_title](#conf-keep-derivations).
+    *Deprecated:* `gc-keep-derivations` is now an alias to `keep-derivations`.
 
   - `gc-keep-outputs`  
-    *Deprecated:* `gc-keep-outputs` is now an alias to
-    [varlistentry\_title](#conf-keep-outputs).
+    *Deprecated:* `gc-keep-outputs` is now an alias to `keep-outputs`.
 
   - `env-keep-derivations`  
-    *Deprecated:* `env-keep-derivations` is now an alias to
-    [varlistentry\_title](#conf-keep-env-derivations).
+    *Deprecated:* `env-keep-derivations` is now an alias to `keep-env-derivations`.
 
   - `extra-binary-caches`  
-    *Deprecated:* `extra-binary-caches` is now an alias to
-    [varlistentry\_title](#conf-extra-substituters).
+    *Deprecated:* `extra-binary-caches` is now an alias to `extra-substituters`.
 
   - `trusted-binary-caches`  
-    *Deprecated:* `trusted-binary-caches` is now an alias to
-    [varlistentry\_title](#conf-trusted-substituters).
+    *Deprecated:* `trusted-binary-caches` is now an alias to `trusted-substituters`.
diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index 14e08e0f1..e5fd45a7f 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -92,9 +92,10 @@ Most Nix commands interpret the following environment variables:
   - `NIX_REMOTE`  
     This variable should be set to `daemon` if you want to use the Nix
     daemon to execute Nix operations. This is necessary in [multi-user
-    Nix installations](#ssec-multi-user). If the Nix daemon's Unix
-    socket is at some non-standard path, this variable should be set to
-    `unix://path/to/socket`. Otherwise, it should be left unset.
+    Nix installations](../installation/multi-user.md). If the Nix
+    daemon's Unix socket is at some non-standard path, this variable
+    should be set to `unix://path/to/socket`. Otherwise, it should be
+    left unset.
 
   - `NIX_SHOW_STATS`  
     If set to `1`, Nix will print some evaluation statistics, such as
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 81dc26a39..026495c8d 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -32,9 +32,10 @@ to a temporary location. The tarball must include a single top-level
 directory containing at least a file named `default.nix`.
 
 `nix-build` is essentially a wrapper around
-[`nix-instantiate`](#sec-nix-instantiate) (to translate a high-level Nix
+[`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
 expression to a low-level store derivation) and [`nix-store
---realise`](#rsec-nix-store-realise) (to build the store derivation).
+--realise`](nix-store.md#operation---realise) (to build the store
+derivation).
 
 > **Warning**
 >
diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index 7b246f3b3..37587c7e1 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -11,16 +11,16 @@ Title: nix-collect-garbage
 # Description
 
 The command `nix-collect-garbage` is mostly an alias of [`nix-store
---gc`](#rsec-nix-store-gc), that is, it deletes all unreachable paths in
-the Nix store to clean up your system. However, it provides two
-additional options: `-d` (`--delete-old`), which deletes all old
-generations of all profiles in `/nix/var/nix/profiles` by invoking
-`nix-env --delete-generations old` on all profiles (of course, this
-makes rollbacks to previous configurations impossible); and
-`--delete-older-than` *period*, where period is a value such as `30d`,
-which deletes all generations older than the specified number of days in
-all profiles in `/nix/var/nix/profiles` (except for the generations that
-were active at that point in time).
+--gc`](nix-store.md#operation---gc), that is, it deletes all
+unreachable paths in the Nix store to clean up your system. However,
+it provides two additional options: `-d` (`--delete-old`), which
+deletes all old generations of all profiles in `/nix/var/nix/profiles`
+by invoking `nix-env --delete-generations old` on all profiles (of
+course, this makes rollbacks to previous configurations impossible);
+and `--delete-older-than` *period*, where period is a value such as
+`30d`, which deletes all generations older than the specified number
+of days in all profiles in `/nix/var/nix/profiles` (except for the
+generations that were active at that point in time).
 
 # Example
 
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index ab1d10c08..c5fcce316 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -92,7 +92,7 @@ have an effect.
     done if this flag had not been specified, without actually doing it.
 
     `--dry-run` also prints out which paths will be
-    [substituted](#gloss-substitute) (i.e., downloaded) and which paths
+    [substituted](../glossary.md) (i.e., downloaded) and which paths
     will be built from source (because no substitute is available).
 
   - `--system-filter` *system*  
@@ -186,11 +186,11 @@ a number of possible ways:
     gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will
     probably cause a user environment conflict\!).
 
-  - If [`--attr`](#opt-attr) (`-A`) is specified, the arguments are
-    *attribute paths* that select attributes from the top-level Nix
+  - If `--attr` (`-A`) is specified, the arguments are *attribute
+    paths* that select attributes from the top-level Nix
     expression. This is faster than using derivation names and
-    unambiguous. To find out the attribute paths of available packages,
-    use `nix-env -qaP`.
+    unambiguous. To find out the attribute paths of available
+    packages, use `nix-env -qaP`.
 
   - If `--from-profile` *path* is given, *args* is a set of names
     denoting installed store paths in the profile *path*. This is an
@@ -198,18 +198,19 @@ a number of possible ways:
     another.
 
   - If `--from-expression` is given, *args* are Nix
-    [functions](#ss-functions) that are called with the active Nix
-    expression as their single argument. The derivations returned by
-    those function calls are installed. This allows derivations to be
-    specified in an unambiguous way, which is necessary if there are
-    multiple derivations with the same name.
+    [functions](../expressions/language-constructs.md#functions)
+    that are called with the active Nix expression as their single
+    argument. The derivations returned by those function calls are
+    installed. This allows derivations to be specified in an
+    unambiguous way, which is necessary if there are multiple
+    derivations with the same name.
 
   - If *args* are store derivations, then these are
-    [realised](#rsec-nix-store-realise), and the resulting output paths
+    [realised](nix-store.md#operation---realise), and the resulting output paths
     are installed.
 
   - If *args* are store paths that are not store derivations, then these
-    are [realised](#rsec-nix-store-realise) and installed.
+    are [realised](nix-store.md#operation---realise) and installed.
 
   - By default all outputs are installed for each derivation. That can
     be reduced by setting `meta.outputsToInstall`.
@@ -319,9 +320,9 @@ left untouched; this is not an error. It is also not an error if an
 element of *args* matches no installed derivations.
 
 For a description of how *args* is mapped to a set of store paths, see
-[`--install`](#rsec-nix-env-install). If *args* describes multiple store
-paths with the same symbolic name, only the one with the highest version
-is installed.
+[`--install`](#operation---install). If *args* describes multiple
+store paths with the same symbolic name, only the one with the highest
+version is installed.
 
 ## Flags
 
@@ -584,9 +585,9 @@ derivation is shown unless `--no-name` is specified.
 
   - `--attr-path`; `-P`  
     Print the *attribute path* of the derivation, which can be used to
-    unambiguously select it using the [`--attr` option](#opt-attr)
-    available in commands that install derivations like `nix-env
-    --install`. This option only works together with `--available`
+    unambiguously select it using the `--attr` option available in
+    commands that install derivations like `nix-env --install`. This
+    option only works together with `--available`
 
   - `--no-name`  
     Suppress printing of the `name` attribute of each derivation.
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index 38c9e0f67..edb331e1c 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -21,13 +21,13 @@ is printed in hexadecimal. To generate the same hash as
 `nix-prefetch-url` you have to specify multiple arguments, see below for
 an example.
 
-The hash is computed over a *serialisation* of each path: a dump of the
-file system tree rooted at the path. This allows directories and
-symlinks to be hashed as well as regular files. The dump is in the *NAR
-format* produced by [`nix-store` `--dump`](#refsec-nix-store-dump).
-Thus, `nix-hash
-path` yields the same cryptographic hash as `nix-store --dump
-path | md5sum`.
+The hash is computed over a *serialisation* of each path: a dump of
+the file system tree rooted at the path. This allows directories and
+symlinks to be hashed as well as regular files. The dump is in the
+*NAR format* produced by [`nix-store
+--dump`](nix-store.md#operation---dump).  Thus, `nix-hash path`
+yields the same cryptographic hash as `nix-store --dump path |
+md5sum`.
 
 # Options
 
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index ca9ef1fc9..2d6525e77 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -21,11 +21,11 @@ Title: nix-instantiate
 # Description
 
 The command `nix-instantiate` generates [store
-derivations](#gloss-derivation) from (high-level) Nix expressions. It
+derivations](../glossary.md) from (high-level) Nix expressions. It
 evaluates the Nix expressions in each of *files* (which defaults to
 *./default.nix*). Each top-level expression should evaluate to a
-derivation, a list of derivations, or a set of derivations. The paths of
-the resulting store derivations are printed on standard output.
+derivation, a list of derivations, or a set of derivations. The paths
+of the resulting store derivations are printed on standard output.
 
 If *files* is the character `-`, then a Nix expression will be read from
 standard input.
@@ -33,7 +33,7 @@ standard input.
 # Options
 
   - `--add-root` *path*; `--indirect`  
-    See the [corresponding options](#opt-add-root) in `nix-store`.
+    See the [corresponding options](nix-store.md) in `nix-store`.
 
   - `--parse`  
     Just parse the input files, and print their abstract syntax trees on
@@ -69,7 +69,7 @@ standard input.
     When used with `--eval`, print the resulting value as an XML
     representation of the abstract syntax tree rather than as an ATerm.
     The schema is the same as that used by the [`toXML`
-    built-in](#builtin-toXML).
+    built-in](../expressions/builtins.md).
 
   - `--read-write-mode`  
     When used with `--eval`, perform evaluation in read/write mode so
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index b15340fde..1842f8858 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -79,13 +79,14 @@ The operation `--realise` essentially “builds” the specified store
 paths. Realisation is a somewhat overloaded term:
 
   - If the store path is a *derivation*, realisation ensures that the
-    output paths of the derivation are [valid](#gloss-validity) (i.e.,
-    the output path and its closure exist in the file system). This can
-    be done in several ways. First, it is possible that the outputs are
-    already valid, in which case we are done immediately. Otherwise,
-    there may be [substitutes](#gloss-substitute) that produce the
-    outputs (e.g., by downloading them). Finally, the outputs can be
-    produced by performing the build action described by the derivation.
+    output paths of the derivation are [valid](../glossary.md) (i.e.,
+    the output path and its closure exist in the file system). This
+    can be done in several ways. First, it is possible that the
+    outputs are already valid, in which case we are done
+    immediately. Otherwise, there may be [substitutes](../glossary.md)
+    that produce the outputs (e.g., by downloading them). Finally, the
+    outputs can be produced by performing the build action described
+    by the derivation.
 
   - If the store path is not a derivation, realisation ensures that the
     specified path is valid (i.e., it and its closure exist in the file
@@ -129,11 +130,12 @@ Special exit codes:
 
   - `101`  
     Build timeout, the build was aborted because it did not complete
-    within the specified [`timeout`](#conf-timeout).
+    within the specified `timeout`.
 
   - `102`  
     Hash mismatch, the build output was rejected because it does not
-    match the specified [`outputHash`](#fixed-output-drvs).
+    match the [`outputHash` attribute of the
+    derivation](../expressions/advanced-attributes.md).
 
   - `104`  
     Not deterministic, the build succeeded in check mode but the
@@ -153,12 +155,12 @@ or.
 ## Examples
 
 This operation is typically used to build store derivations produced by
-[`nix-instantiate`](#sec-nix-instantiate):
+[`nix-instantiate`](nix-instantiate.md):
 
     $ nix-store -r $(nix-instantiate ./test.nix)
     /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
 
-This is essentially what [`nix-build`](#sec-nix-build) does.
+This is essentially what [`nix-build`](nix-build.md) does.
 
 To test whether a previously-built derivation is deterministic:
 
@@ -232,8 +234,7 @@ control what gets deleted and in what order:
     or TiB units.
 
 The behaviour of the collector is also influenced by the
-[`keep-outputs`](#conf-keep-outputs) and
-[`keep-derivations`](#conf-keep-derivations) variables in the Nix
+`keep-outputs` and `keep-derivations` variables in the Nix
 configuration file.
 
 By default, the collector prints the total number of freed bytes when it
@@ -307,17 +308,17 @@ symlink.
 
   - `--force-realise`; `-f`  
     Realise each argument to the query first (see [`nix-store
-                    --realise`](#rsec-nix-store-realise)).
+    --realise`](#operation---realise)).
 
 ## Queries
 
   - `--outputs`  
-    Prints out the [output paths](#gloss-output-path) of the store
+    Prints out the [output paths](../glossary.md) of the store
     derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
   - `--requisites`; `-R`  
-    Prints out the [closure](#gloss-closure) of the store path *paths*.
+    Prints out the [closure](../glossary.md) of the store path *paths*.
 
     This query has one option:
 
@@ -334,7 +335,7 @@ symlink.
     derivation and specifying the option `--include-outputs`.
 
   - `--references`  
-    Prints the set of [references](#gloss-reference) of the store paths
+    Prints the set of [references](../glossary.md) of the store paths
     *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
@@ -352,7 +353,7 @@ symlink.
     in the Nix store that are dependent on *paths*.
 
   - `--deriver`; `-d`  
-    Prints the [deriver](#gloss-deriver) of the store paths *paths*. If
+    Prints the [deriver](../glossary.md) of the store paths *paths*. If
     the path has no deriver (e.g., if it is a source file), or if the
     deriver is not known (e.g., in the case of a binary-only
     deployment), the string `unknown-deriver` is printed.
@@ -605,13 +606,12 @@ anyway. Likewise, all permissions are left out except for the execute
 bit, because all files in the Nix store have 444 or 555 permission.
 
 Also, a NAR archive is *canonical*, meaning that “equal” paths always
-produce the same NAR archive. For instance, directory entries are always
-sorted so that the actual on-disk order doesn’t influence the result.
-This means that the cryptographic hash of a NAR dump of a path is usable
-as a fingerprint of the contents of the path. Indeed, the hashes of
-store paths stored in Nix’s database (see [`nix-store -q
---hash`](#refsec-nix-store-query)) are SHA-256 hashes of the NAR dump of
-each store path.
+produce the same NAR archive. For instance, directory entries are
+always sorted so that the actual on-disk order doesn’t influence the
+result.  This means that the cryptographic hash of a NAR dump of a
+path is usable as a fingerprint of the contents of the path. Indeed,
+the hashes of store paths stored in Nix’s database (see `nix-store -q
+--hash`) are SHA-256 hashes of the NAR dump of each store path.
 
 NAR archives support filenames of unlimited length and 64-bit file
 sizes. They can contain regular files, directories, and symbolic links,
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index b9c65c81d..ee8419fd2 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -71,35 +71,34 @@ Most Nix commands accept the following command-line options:
   - `--max-jobs` / `-j` *number*  
     Sets the maximum number of build jobs that Nix will perform in
     parallel to the specified number. Specify `auto` to use the number
-    of CPUs in the system. The default is specified by the
-    [`max-jobs`](#conf-max-jobs) configuration setting, which itself
-    defaults to `1`. A higher value is useful on SMP systems or to
-    exploit I/O latency.
+    of CPUs in the system. The default is specified by the `max-jobs`
+    configuration setting, which itself defaults to `1`. A higher
+    value is useful on SMP systems or to exploit I/O latency.
     
     Setting it to `0` disallows building on the local machine, which is
     useful when you want builds to happen only on remote builders.
 
   - `--cores`  
-    Sets the value of the `NIX_BUILD_CORES` environment variable in the
-    invocation of builders. Builders can use this variable at their
-    discretion to control the maximum amount of parallelism. For
+    Sets the value of the `NIX_BUILD_CORES` environment variable in
+    the invocation of builders. Builders can use this variable at
+    their discretion to control the maximum amount of parallelism. For
     instance, in Nixpkgs, if the derivation attribute
     `enableParallelBuilding` is set to `true`, the builder passes the
-    `-jN` flag to GNU Make. It defaults to the value of the
-    [`cores`](#conf-cores) configuration setting, if set, or `1`
-    otherwise. The value `0` means that the builder should use all
-    available CPU cores in the system.
+    `-jN` flag to GNU Make. It defaults to the value of the `cores`
+    configuration setting, if set, or `1` otherwise. The value `0`
+    means that the builder should use all available CPU cores in the
+    system.
 
   - `--max-silent-time`  
     Sets the maximum number of seconds that a builder can go without
-    producing any data on standard output or standard error. The default
-    is specified by the [`max-silent-time`](#conf-max-silent-time)
-    configuration setting. `0` means no time-out.
+    producing any data on standard output or standard error. The
+    default is specified by the `max-silent-time` configuration
+    setting. `0` means no time-out.
 
   - `--timeout`  
     Sets the maximum number of seconds that a builder can run. The
-    default is specified by the [`timeout`](#conf-timeout) configuration
-    setting. `0` means no timeout.
+    default is specified by the `timeout` configuration setting. `0`
+    means no timeout.
 
   - `--keep-going` / `-k`  
     Keep going in case of failed builds, to the greatest extent
@@ -145,16 +144,17 @@ Most Nix commands accept the following command-line options:
     operations will fail.
 
   - `--arg` *name* *value*  
-    This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell`
-    and `nix-build`. When evaluating Nix expressions, the expression
-    evaluator will automatically try to call functions that it
-    encounters. It can automatically call functions for which every
-    argument has a [default value](#ss-functions) (e.g., `{ argName ?
-            defaultValue }:
-            ...`). With `--arg`, you can also call functions that have arguments
-    without a default value (or override a default value). That is, if
-    the evaluator encounters a function with an argument named *name*,
-    it will call it with value *value*.
+    This option is accepted by `nix-env`, `nix-instantiate`,
+    `nix-shell` and `nix-build`. When evaluating Nix expressions, the
+    expression evaluator will automatically try to call functions that
+    it encounters. It can automatically call functions for which every
+    argument has a [default
+    value](../expressions/language-constructs.md#functions) (e.g.,
+    `{ argName ?  defaultValue }: ...`). With `--arg`, you can also
+    call functions that have arguments without a default value (or
+    override a default value). That is, if the evaluator encounters a
+    function with an argument named *name*, it will call it with value
+    *value*.
     
     For instance, the top-level `default.nix` in Nixpkgs is actually a
     function:
@@ -165,28 +165,28 @@ Most Nix commands accept the following command-line options:
         }: ...
     
     So if you call this Nix expression (e.g., when you do `nix-env -i
-    pkgname`), the function will be called automatically using the value
-    [`builtins.currentSystem`](#builtin-currentSystem) for the `system`
-    argument. You can override this using `--arg`, e.g., `nix-env -i
-            pkgname --arg system
-            \"i686-freebsd\"`. (Note that since the argument is a Nix string
-    literal, you have to escape the quotes.)
+    pkgname`), the function will be called automatically using the
+    value [`builtins.currentSystem`](../expressions/builtins.md) for
+    the `system` argument. You can override this using `--arg`, e.g.,
+    `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that
+    since the argument is a Nix string literal, you have to escape the
+    quotes.)
 
   - `--argstr` *name* *value*  
-    This option is like `--arg`, only the value is not a Nix expression
-    but a string. So instead of `--arg system \"i686-linux\"` (the outer
-    quotes are to keep the shell happy) you can say `--argstr system
-            i686-linux`.
+    This option is like `--arg`, only the value is not a Nix
+    expression but a string. So instead of `--arg system
+    \"i686-linux\"` (the outer quotes are to keep the shell happy) you
+    can say `--argstr system i686-linux`.
 
   - `--attr` / `-A` *attrPath*  
     Select an attribute from the top-level Nix expression being
     evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and
-    `nix-shell` only.) The *attribute path* *attrPath* is a sequence of
-    attribute names separated by dots. For instance, given a top-level
-    Nix expression *e*, the attribute path `xorg.xorgserver` would cause
-    the expression `e.xorg.xorgserver` to be used. See [`nix-env
-            --install`](#refsec-nix-env-install-examples) for some concrete
-    examples.
+    `nix-shell` only.) The *attribute path* *attrPath* is a sequence
+    of attribute names separated by dots. For instance, given a
+    top-level Nix expression *e*, the attribute path `xorg.xorgserver`
+    would cause the expression `e.xorg.xorgserver` to be used. See
+    [`nix-env --install`](nix-env.md#operation---install) for some
+    concrete examples.
     
     In addition to attribute names, you can also specify array indices.
     For instance, the attribute path `foo.3.bar` selects the `bar`
diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
index 3582fa268..bfee28acf 100644
--- a/doc/manual/src/expressions/advanced-attributes.md
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -89,10 +89,10 @@ Derivations can declare some infrequently used optional attributes.
     to make it use the proxy server configuration specified by the user
     in the environment variables `http_proxy` and friends.
     
-    This attribute is only allowed in [fixed-output
-    derivations](#fixed-output-drvs), where impurities such as these are
-    okay since (the hash of) the output is known in advance. It is
-    ignored for all other derivations.
+    This attribute is only allowed in *fixed-output derivations* (see
+    below), where impurities such as these are okay since (the hash
+    of) the output is known in advance. It is ignored for all other
+    derivations.
     
     > **Warning**
     > 
@@ -183,13 +183,14 @@ Derivations can declare some infrequently used optional attributes.
       - `"recursive"`  
         The hash is computed over the NAR archive dump of the output
         (i.e., the result of [`nix-store
-                                                                        --dump`](#refsec-nix-store-dump)). In this case, the output can
-        be anything, including a directory tree.
+        --dump`](../command-ref/nix-store.md#operation---dump)). In
+        this case, the output can be anything, including a directory
+        tree.
     
-    The `outputHash` attribute, finally, must be a string containing the
-    hash in either hexadecimal or base-32 notation. (See the [`nix-hash`
-    command](#sec-nix-hash) for information about converting to and from
-    base-32 notation.)
+    The `outputHash` attribute, finally, must be a string containing
+    the hash in either hexadecimal or base-32 notation. (See the
+    [`nix-hash` command](../command-ref/nix-hash.md) for information
+    about converting to and from base-32 notation.)
 
   - `passAsFile`  
     A list of names of attributes that should be passed via files rather
@@ -213,10 +214,11 @@ Derivations can declare some infrequently used optional attributes.
 
   - `preferLocalBuild`  
     If this attribute is set to `true` and [distributed building is
-    enabled](#chap-distributed-builds), then, if possible, the derivaton
-    will be built locally instead of forwarded to a remote machine. This
-    is appropriate for trivial builders where the cost of doing a
-    download or remote build would exceed the cost of building locally.
+    enabled](../advanced-topics/distributed-builds.md), then, if
+    possible, the derivaton will be built locally instead of forwarded
+    to a remote machine. This is appropriate for trivial builders
+    where the cost of doing a download or remote build would exceed
+    the cost of building locally.
 
   - `allowSubstitutes`  
     If this attribute is set to `false`, then Nix will always build this
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index 3c9bb4533..22f133c33 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -57,19 +57,19 @@ For instance, `derivation` is also available as `builtins.derivation`.
     installations that don’t have the desired built-in function.
 
   - `builtins.compareVersions` *s1* *s2*  
-    Compare two strings representing versions and return `-1` if version
-    *s1* is older than version *s2*, `0` if they are the same, and `1`
-    if *s1* is newer than *s2*. The version comparison algorithm is the
-    same as the one used by [`nix-env
-                    -u`](#ssec-version-comparisons).
+    Compare two strings representing versions and return `-1` if
+    version *s1* is older than version *s2*, `0` if they are the same,
+    and `1` if *s1* is newer than *s2*. The version comparison
+    algorithm is the same as the one used by [`nix-env
+    -u`](../command-ref/nix-env.md#operation---upgrade).
 
   - `builtins.concatLists` *lists*  
     Concatenate a list of lists into a single list.
 
   - `builtins.concatStringsSep` *separator* *list*  
-    Concatenate a list of strings with a separator between each element,
-    e.g. `concatStringsSep "/"
-                    ["usr" "local" "bin"] == "usr/local/bin"`
+    Concatenate a list of strings with a separator between each
+    element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] ==
+    "usr/local/bin"`
 
   - `builtins.currentSystem`  
     The built-in value `currentSystem` evaluates to the Nix platform
@@ -77,10 +77,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
     evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
 
   - `builtins.deepSeq` *e1* *e2*  
-    This is like `seq
-                    e1
-                    e2`, except that *e1* is evaluated *deeply*: if it’s a list or set,
-    its elements or attributes are also evaluated recursively.
+    This is like `seq e1 e2`, except that *e1* is evaluated *deeply*:
+    if it’s a list or set, its elements or attributes are also
+    evaluated recursively.
 
   - `derivation` *attrs*; `builtins.derivation` *attrs*  
     `derivation` is described in [its own section](derivations.md).
@@ -104,7 +103,7 @@ For instance, `derivation` is also available as `builtins.derivation`.
   - `builtins.fetchurl` *url*  
     Download the specified URL and return the path of the downloaded
     file. This function is not available if [restricted evaluation
-    mode](#conf-restrict-eval) is enabled.
+    mode](../command-ref/conf-file.md) is enabled.
 
   - `fetchTarball` *url*; `builtins.fetchTarball` *url*  
     Download the specified URL, unpack it and return the path of the
@@ -140,7 +139,7 @@ For instance, `derivation` is also available as `builtins.derivation`.
         stdenv.mkDerivation { … }
     
     This function is not available if [restricted evaluation
-    mode](#conf-restrict-eval) is enabled.
+    mode](../command-ref/conf-file.md) is enabled.
 
   - `builtins.fetchGit` *args*  
     Fetch a path from git. *args* can be a URL, in which case the HEAD
@@ -491,9 +490,8 @@ For instance, `derivation` is also available as `builtins.derivation`.
     name is everything up to but not including the first dash followed
     by a digit, and the version is everything following that dash. The
     result is returned in a set `{ name, version }`. Thus,
-    `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = "nix";
-    version = "0.12pre12876";
-                    }`.
+    `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name =
+    "nix"; version = "0.12pre12876"; }`.
 
   - `builtins.path` *args*  
     An enrichment of the built-in path type, based on the attributes
@@ -508,9 +506,8 @@ For instance, `derivation` is also available as `builtins.derivation`.
         like `@`.
     
       - filter  
-        A function of the type expected by
-        [builtins.filterSource](#builtin-filterSource), with the same
-        semantics.
+        A function of the type expected by `builtins.filterSource`,
+        with the same semantics.
     
       - recursive  
         When `false`, when `path` is added to the store it is with a
@@ -609,7 +606,7 @@ For instance, `derivation` is also available as `builtins.derivation`.
   - `builtins.splitVersion` *s*  
     Split a string representing a version into its components, by the
     same version splitting logic underlying the version comparison in
-    [`nix-env -u`](#ssec-version-comparisons).
+    [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade).
 
   - `builtins.stringLength` *e*  
     Return the length of the string *e*. If *e* is not a string,
diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md
index c3e8d7f82..ca422acea 100644
--- a/doc/manual/src/expressions/simple-building-testing.md
+++ b/doc/manual/src/expressions/simple-building-testing.md
@@ -19,11 +19,11 @@ yet. The best way to test the package is by using the command
     $ ./result/bin/hello
     Hello, world!
 
-The [`-A`](#opt-attr) option selects the `hello` attribute. This is
-faster than using the symbolic package name specified by the `name`
-attribute (which also happens to be `hello`) and is unambiguous (there
-can be multiple packages with the symbolic name `hello`, but there can
-be only one attribute in a set named `hello`).
+The `-A` option selects the `hello` attribute. This is faster than
+using the symbolic package name specified by the `name` attribute
+(which also happens to be `hello`) and is unambiguous (there can be
+multiple packages with the symbolic name `hello`, but there can be
+only one attribute in a set named `hello`).
 
 `nix-build` registers the `./result` symlink as a garbage collection
 root, so unless and until you delete the `./result` symlink, the output
diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index 6983452d3..a1dd993c4 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -8,7 +8,7 @@ easiest way to install Nix is to run the following command:
 ```
 
 If you're using macOS 10.15 (Catalina) or newer, consult [the macOS
-installation instructions](#sect-macos-installation) before installing.
+installation instructions](#macos-installation) before installing.
 
 As of Nix 2.1.0, the Nix installer will always default to creating a
 single-user installation, however opting in to the multi-user
@@ -64,7 +64,7 @@ will invoke `sudo` as needed.
 > 
 > If you need Nix to use a different group ID or user ID set, you will
 > have to download the tarball manually and [edit the install
-> script](#sect-nix-install-binary-tarball).
+> script](#installing-from-a-binary-tarball).
 
 The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
 The installer will first back up these files with a `.backup-before-nix`
diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/src/installation/multi-user.md
index 4a861691d..de159b603 100644
--- a/doc/manual/src/installation/multi-user.md
+++ b/doc/manual/src/installation/multi-user.md
@@ -39,16 +39,16 @@ expect to do many builds at the same time.
 
 ## Running the daemon
 
-The [Nix daemon](#sec-nix-daemon) should be started as follows (as
-`root`):
+The [Nix daemon](../command-ref/nix-daemon.md) should be started as
+follows (as `root`):
 
     $ nix-daemon
 
 You’ll want to put that line somewhere in your system’s boot scripts.
 
 To let unprivileged users use the daemon, they should set the
-[`NIX_REMOTE` environment variable](#envar-remote) to `daemon`. So you
-should put a line like
+[`NIX_REMOTE` environment variable](../command-ref/env-common.md) to
+`daemon`. So you should put a line like
 
     export NIX_REMOTE=daemon
 
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
index 8edaca6b0..17b5cc9c2 100644
--- a/doc/manual/src/package-management/basic-package-mgmt.md
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -1,8 +1,9 @@
 # Basic Package Management
 
-The main command for package management is [`nix-env`](#sec-nix-env).
-You can use it to install, upgrade, and erase packages, and to query
-what packages are installed or are available for installation.
+The main command for package management is
+[`nix-env`](../command-ref/nix-env.md).  You can use it to install,
+upgrade, and erase packages, and to query what packages are installed
+or are available for installation.
 
 In Nix, different users can have different “views” on the set of
 installed applications. That is, there might be lots of applications
diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md
index 8a3e92d70..c239998d9 100644
--- a/doc/manual/src/package-management/channels.md
+++ b/doc/manual/src/package-management/channels.md
@@ -7,8 +7,8 @@ better way: *Nix channels*.
 
 A Nix channel is just a URL that points to a place that contains a set
 of Nix expressions and a manifest. Using the command
-[`nix-channel`](#sec-nix-channel) you can automatically stay up to date
-with whatever is available at that URL.
+[`nix-channel`](../command-ref/nix-channel.md) you can automatically
+stay up to date with whatever is available at that URL.
 
 To see the list of official NixOS channels, visit
 <https://nixos.org/channels>.
diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md
index d96114e3c..2824c1a9b 100644
--- a/doc/manual/src/package-management/s3-substituter.md
+++ b/doc/manual/src/package-management/s3-substituter.md
@@ -1,9 +1,9 @@
 # Serving a Nix store via S3
 
 Nix has built-in support for storing and fetching store paths from
-Amazon S3 and S3-compatible services. This uses the same *binary* cache
-mechanism that Nix usually uses to fetch prebuilt binaries from
-[cache.nixos.org](cache.nixos.org).
+Amazon S3 and S3-compatible services. This uses the same *binary*
+cache mechanism that Nix usually uses to fetch prebuilt binaries from
+[cache.nixos.org](https://cache.nixos.org/).
 
 The following options can be specified as URL parameters to the S3 URL:
 
@@ -85,8 +85,8 @@ caches.
 
 Your bucket will need a bucket policy allowing the desired users to
 perform the `s3:GetObject` and `s3:GetBucketLocation` action on all
-objects in the bucket. The anonymous policy in [Anonymous Reads to your
-S3-compatible binary cache](#ssec-s3-substituter-anonymous-reads) can be
+objects in the bucket. The [anonymous policy given
+above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be
 updated to have a restricted `Principal` to support this.
 
 ## Authenticated Writes to your S3-compatible binary cache
diff --git a/doc/manual/src/release-notes/rl-2.0.md b/doc/manual/src/release-notes/rl-2.0.md
index 170275f89..9f6d4aa83 100644
--- a/doc/manual/src/release-notes/rl-2.0.md
+++ b/doc/manual/src/release-notes/rl-2.0.md
@@ -148,11 +148,11 @@ This release has the following new features:
         `nix-store
                                                                         --verify-path`.
     
-      - `nix log` shows the build log of a package or path. If the build
-        log is not available locally, it will try to obtain it from the
-        configured substituters (such as
-        [cache.nixos.org](cache.nixos.org), which now provides build
-        logs).
+      - `nix log` shows the build log of a package or path. If the
+        build log is not available locally, it will try to obtain it
+        from the configured substituters (such as
+        [cache.nixos.org](https://cache.nixos.org/), which now
+        provides build logs).
     
       - `nix edit` opens the source code of a package in your editor.
     
@@ -213,16 +213,17 @@ This release has the following new features:
         current values.
 
   - The store abstraction that Nix has had for a long time to support
-    store access via the Nix daemon has been extended significantly. In
-    particular, substituters (which used to be external programs such as
-    `download-from-binary-cache`) are now subclasses of the abstract
-    `Store` class. This allows many Nix commands to operate on such
-    store types. For example, `nix path-info` shows information about
-    paths in your local Nix store, while `nix path-info --store
-                    https://cache.nixos.org/` shows information about paths in the
-    specified binary cache. Similarly, `nix-copy-closure`, `nix-push`
-    and substitution are all instances of the general notion of copying
-    paths between different kinds of Nix stores.
+    store access via the Nix daemon has been extended
+    significantly. In particular, substituters (which used to be
+    external programs such as `download-from-binary-cache`) are now
+    subclasses of the abstract `Store` class. This allows many Nix
+    commands to operate on such store types. For example, `nix
+    path-info` shows information about paths in your local Nix store,
+    while `nix path-info --store https://cache.nixos.org/` shows
+    information about paths in the specified binary cache. Similarly,
+    `nix-copy-closure`, `nix-push` and substitution are all instances
+    of the general notion of copying paths between different kinds of
+    Nix stores.
     
     Stores are specified using an URI-like syntax, e.g.
     <https://cache.nixos.org/> or <ssh://machine>. The following store
@@ -241,7 +242,7 @@ This release has the following new features:
         `/home/alice/nix/store`) to differ from its “logical” location
         (typically `/nix/store`). This allows non-root users to use Nix
         while still getting the benefits from prebuilt binaries from
-        [cache.nixos.org](cache.nixos.org).
+        [cache.nixos.org](https://cache.nixos.org/).
     
       - `BinaryCacheStore` is the abstract superclass of all binary
         cache stores. It supports writing build logs and NAR content
@@ -356,11 +357,11 @@ This release has the following new features:
   - `NIX_PATH` is now lazy, so URIs in the path are only downloaded if
     they are needed for evaluation.
 
-  - You can now use <channel:> as a short-hand for
+  - You can now use `channel:` as a short-hand for
     <https://nixos.org/channels//nixexprs.tar.xz>. For example,
     `nix-build channel:nixos-15.09 -A hello` will build the GNU Hello
-    package from the `nixos-15.09` channel. In the future, this may use
-    Git to fetch updates more efficiently.
+    package from the `nixos-15.09` channel. In the future, this may
+    use Git to fetch updates more efficiently.
 
   - When `--no-build-output` is given, the last 10 lines of the build
     log will be shown if a build fails.
@@ -382,7 +383,7 @@ This release has the following new features:
         in all places where Nix allows URIs.
     
       - Brotli compression is now supported. In particular,
-        [cache.nixos.org](cache.nixos.org) build logs are now compressed
+        [cache.nixos.org](https://cache.nixos.org/) build logs are now compressed
         using Brotli.
 
   - `nix-env`
diff --git a/doc/manual/src/release-notes/rl-2.1.md b/doc/manual/src/release-notes/rl-2.1.md
index 08986ef9d..b88834c83 100644
--- a/doc/manual/src/release-notes/rl-2.1.md
+++ b/doc/manual/src/release-notes/rl-2.1.md
@@ -4,18 +4,18 @@ This is primarily a bug fix release. It also reduces memory consumption
 in certain situations. In addition, it has the following new features:
 
   - The Nix installer will no longer default to the Multi-User
-    installation for macOS. You can still [instruct the installer to run
-    in multi-user mode](#sect-multi-user-installation).
+    installation for macOS. You can still instruct the installer to
+    run in multi-user mode.
 
-  - The Nix installer now supports performing a Multi-User installation
-    for Linux computers which are running systemd. You can [select a
-    Multi-User installation](#sect-multi-user-installation) by passing
-    the `--daemon` flag to the installer: `sh <(curl
-                    https://nixos.org/nix/install) --daemon`.
-    
-    The multi-user installer cannot handle systems with SELinux. If your
-    system has SELinux enabled, you can [force the installer to run in
-    single-user mode](#sect-single-user-installation).
+  - The Nix installer now supports performing a Multi-User
+    installation for Linux computers which are running systemd. You
+    can select a Multi-User installation by passing the `--daemon`
+    flag to the installer: `sh <(curl https://nixos.org/nix/install)
+    --daemon`.
+
+    The multi-user installer cannot handle systems with SELinux. If
+    your system has SELinux enabled, you can force the installer to
+    run in single-user mode.
 
   - New builtin functions: `builtins.bitAnd`, `builtins.bitOr`,
     `builtins.bitXor`, `builtins.fromTOML`, `builtins.concatMap`,

From 1308c8404e19aacc6458b3813d445857620a60a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 15:48:40 +0200
Subject: [PATCH 045/882] Remove DocBook manual

---
 .gitignore                                    |    5 -
 .../advanced-topics/advanced-topics.xml       |   14 -
 doc/manual/advanced-topics/cores-vs-jobs.xml  |  121 --
 doc/manual/advanced-topics/diff-hook.xml      |  202 --
 .../advanced-topics/distributed-builds.xml    |  190 --
 .../advanced-topics/post-build-hook.xml       |  158 --
 doc/manual/command-ref/command-ref.xml        |   20 -
 doc/manual/command-ref/conf-file.xml          | 1244 ------------
 doc/manual/command-ref/env-common.xml         |  209 ---
 doc/manual/command-ref/files.xml              |   14 -
 doc/manual/command-ref/main-commands.xml      |   17 -
 doc/manual/command-ref/nix-build.xml          |  190 --
 doc/manual/command-ref/nix-channel.xml        |  181 --
 .../command-ref/nix-collect-garbage.xml       |   63 -
 doc/manual/command-ref/nix-copy-closure.xml   |  169 --
 doc/manual/command-ref/nix-daemon.xml         |   35 -
 doc/manual/command-ref/nix-env.xml            | 1503 ---------------
 doc/manual/command-ref/nix-hash.xml           |  176 --
 doc/manual/command-ref/nix-instantiate.xml    |  266 ---
 doc/manual/command-ref/nix-prefetch-url.xml   |  131 --
 doc/manual/command-ref/nix-shell.xml          |  411 ----
 doc/manual/command-ref/nix-store.xml          | 1516 ---------------
 doc/manual/command-ref/opt-common-syn.xml     |   68 -
 doc/manual/command-ref/opt-common.xml         |  405 ----
 doc/manual/command-ref/opt-inst-syn.xml       |   22 -
 doc/manual/command-ref/utilities.xml          |   20 -
 .../expressions/advanced-attributes.xml       |  351 ----
 .../expressions/arguments-variables.xml       |  114 --
 doc/manual/expressions/build-script.xml       |  114 --
 doc/manual/expressions/builtins.xml           | 1663 -----------------
 doc/manual/expressions/derivations.xml        |  210 ---
 .../expressions/expression-language.xml       |   30 -
 doc/manual/expressions/expression-syntax.xml  |  146 --
 doc/manual/expressions/generic-builder.xml    |   98 -
 .../expressions/language-constructs.xml       |  408 ----
 doc/manual/expressions/language-operators.xml |  222 ---
 doc/manual/expressions/language-values.xml    |  312 ----
 .../expressions/simple-building-testing.xml   |   76 -
 doc/manual/expressions/simple-expression.xml  |   46 -
 .../expressions/writing-nix-expressions.xml   |   26 -
 doc/manual/glossary/glossary.xml              |  199 --
 doc/manual/hacking.xml                        |   74 -
 doc/manual/installation/building-source.xml   |   49 -
 doc/manual/installation/env-variables.xml     |   89 -
 doc/manual/installation/installation.xml      |   34 -
 doc/manual/installation/installing-binary.xml |  469 -----
 doc/manual/installation/installing-source.xml |   16 -
 doc/manual/installation/multi-user.xml        |  107 --
 doc/manual/installation/nix-security.xml      |   27 -
 doc/manual/installation/obtaining-source.xml  |   30 -
 .../installation/prerequisites-source.xml     |  101 -
 doc/manual/installation/single-user.xml       |   21 -
 .../installation/supported-platforms.xml      |   36 -
 doc/manual/installation/upgrading.xml         |   27 -
 doc/manual/introduction/about-nix.xml         |  268 ---
 doc/manual/introduction/introduction.xml      |   12 -
 doc/manual/introduction/quick-start.xml       |  124 --
 doc/manual/local.mk                           |    9 -
 doc/manual/manual.xml                         |   52 -
 doc/manual/nix-lang-ref.xml                   |  182 --
 doc/manual/packages/basic-package-mgmt.xml    |  194 --
 .../packages/binary-cache-substituter.xml     |   70 -
 doc/manual/packages/channels.xml              |   60 -
 doc/manual/packages/copy-closure.xml          |   50 -
 doc/manual/packages/garbage-collection.xml    |   86 -
 .../packages/garbage-collector-roots.xml      |   29 -
 doc/manual/packages/package-management.xml    |   23 -
 doc/manual/packages/profiles.xml              |  154 --
 doc/manual/packages/s3-substituter.xml        |  191 --
 doc/manual/packages/sharing-packages.xml      |   20 -
 doc/manual/packages/ssh-substituter.xml       |   73 -
 doc/manual/release-notes/release-notes.xml    |   51 -
 doc/manual/release-notes/rl-0.10.1.xml        |   13 -
 doc/manual/release-notes/rl-0.10.xml          |  323 ----
 doc/manual/release-notes/rl-0.11.xml          |  261 ---
 doc/manual/release-notes/rl-0.12.xml          |  175 --
 doc/manual/release-notes/rl-0.13.xml          |  106 --
 doc/manual/release-notes/rl-0.14.xml          |   46 -
 doc/manual/release-notes/rl-0.15.xml          |   14 -
 doc/manual/release-notes/rl-0.16.xml          |   55 -
 doc/manual/release-notes/rl-0.5.xml           |   11 -
 doc/manual/release-notes/rl-0.6.xml           |  122 --
 doc/manual/release-notes/rl-0.7.xml           |   35 -
 doc/manual/release-notes/rl-0.8.1.xml         |   21 -
 doc/manual/release-notes/rl-0.8.xml           |  246 ---
 doc/manual/release-notes/rl-0.9.1.xml         |   13 -
 doc/manual/release-notes/rl-0.9.2.xml         |   28 -
 doc/manual/release-notes/rl-0.9.xml           |   98 -
 doc/manual/release-notes/rl-1.0.xml           |  119 --
 doc/manual/release-notes/rl-1.1.xml           |  100 -
 doc/manual/release-notes/rl-1.10.xml          |   64 -
 doc/manual/release-notes/rl-1.11.10.xml       |   31 -
 doc/manual/release-notes/rl-1.11.xml          |  141 --
 doc/manual/release-notes/rl-1.2.xml           |  157 --
 doc/manual/release-notes/rl-1.3.xml           |   19 -
 doc/manual/release-notes/rl-1.4.xml           |   39 -
 doc/manual/release-notes/rl-1.5.1.xml         |   12 -
 doc/manual/release-notes/rl-1.5.2.xml         |   12 -
 doc/manual/release-notes/rl-1.5.xml           |   12 -
 doc/manual/release-notes/rl-1.6.1.xml         |   69 -
 doc/manual/release-notes/rl-1.6.xml           |  127 --
 doc/manual/release-notes/rl-1.7.xml           |  263 ---
 doc/manual/release-notes/rl-1.8.xml           |  123 --
 doc/manual/release-notes/rl-1.9.xml           |  216 ---
 doc/manual/release-notes/rl-2.0.xml           | 1012 ----------
 doc/manual/release-notes/rl-2.1.xml           |  133 --
 doc/manual/release-notes/rl-2.2.xml           |  143 --
 doc/manual/release-notes/rl-2.3.xml           |   91 -
 doc/manual/schemas.xml                        |    4 -
 109 files changed, 18547 deletions(-)
 delete mode 100644 doc/manual/advanced-topics/advanced-topics.xml
 delete mode 100644 doc/manual/advanced-topics/cores-vs-jobs.xml
 delete mode 100644 doc/manual/advanced-topics/diff-hook.xml
 delete mode 100644 doc/manual/advanced-topics/distributed-builds.xml
 delete mode 100644 doc/manual/advanced-topics/post-build-hook.xml
 delete mode 100644 doc/manual/command-ref/command-ref.xml
 delete mode 100644 doc/manual/command-ref/conf-file.xml
 delete mode 100644 doc/manual/command-ref/env-common.xml
 delete mode 100644 doc/manual/command-ref/files.xml
 delete mode 100644 doc/manual/command-ref/main-commands.xml
 delete mode 100644 doc/manual/command-ref/nix-build.xml
 delete mode 100644 doc/manual/command-ref/nix-channel.xml
 delete mode 100644 doc/manual/command-ref/nix-collect-garbage.xml
 delete mode 100644 doc/manual/command-ref/nix-copy-closure.xml
 delete mode 100644 doc/manual/command-ref/nix-daemon.xml
 delete mode 100644 doc/manual/command-ref/nix-env.xml
 delete mode 100644 doc/manual/command-ref/nix-hash.xml
 delete mode 100644 doc/manual/command-ref/nix-instantiate.xml
 delete mode 100644 doc/manual/command-ref/nix-prefetch-url.xml
 delete mode 100644 doc/manual/command-ref/nix-shell.xml
 delete mode 100644 doc/manual/command-ref/nix-store.xml
 delete mode 100644 doc/manual/command-ref/opt-common-syn.xml
 delete mode 100644 doc/manual/command-ref/opt-common.xml
 delete mode 100644 doc/manual/command-ref/opt-inst-syn.xml
 delete mode 100644 doc/manual/command-ref/utilities.xml
 delete mode 100644 doc/manual/expressions/advanced-attributes.xml
 delete mode 100644 doc/manual/expressions/arguments-variables.xml
 delete mode 100644 doc/manual/expressions/build-script.xml
 delete mode 100644 doc/manual/expressions/builtins.xml
 delete mode 100644 doc/manual/expressions/derivations.xml
 delete mode 100644 doc/manual/expressions/expression-language.xml
 delete mode 100644 doc/manual/expressions/expression-syntax.xml
 delete mode 100644 doc/manual/expressions/generic-builder.xml
 delete mode 100644 doc/manual/expressions/language-constructs.xml
 delete mode 100644 doc/manual/expressions/language-operators.xml
 delete mode 100644 doc/manual/expressions/language-values.xml
 delete mode 100644 doc/manual/expressions/simple-building-testing.xml
 delete mode 100644 doc/manual/expressions/simple-expression.xml
 delete mode 100644 doc/manual/expressions/writing-nix-expressions.xml
 delete mode 100644 doc/manual/glossary/glossary.xml
 delete mode 100644 doc/manual/hacking.xml
 delete mode 100644 doc/manual/installation/building-source.xml
 delete mode 100644 doc/manual/installation/env-variables.xml
 delete mode 100644 doc/manual/installation/installation.xml
 delete mode 100644 doc/manual/installation/installing-binary.xml
 delete mode 100644 doc/manual/installation/installing-source.xml
 delete mode 100644 doc/manual/installation/multi-user.xml
 delete mode 100644 doc/manual/installation/nix-security.xml
 delete mode 100644 doc/manual/installation/obtaining-source.xml
 delete mode 100644 doc/manual/installation/prerequisites-source.xml
 delete mode 100644 doc/manual/installation/single-user.xml
 delete mode 100644 doc/manual/installation/supported-platforms.xml
 delete mode 100644 doc/manual/installation/upgrading.xml
 delete mode 100644 doc/manual/introduction/about-nix.xml
 delete mode 100644 doc/manual/introduction/introduction.xml
 delete mode 100644 doc/manual/introduction/quick-start.xml
 delete mode 100644 doc/manual/manual.xml
 delete mode 100644 doc/manual/nix-lang-ref.xml
 delete mode 100644 doc/manual/packages/basic-package-mgmt.xml
 delete mode 100644 doc/manual/packages/binary-cache-substituter.xml
 delete mode 100644 doc/manual/packages/channels.xml
 delete mode 100644 doc/manual/packages/copy-closure.xml
 delete mode 100644 doc/manual/packages/garbage-collection.xml
 delete mode 100644 doc/manual/packages/garbage-collector-roots.xml
 delete mode 100644 doc/manual/packages/package-management.xml
 delete mode 100644 doc/manual/packages/profiles.xml
 delete mode 100644 doc/manual/packages/s3-substituter.xml
 delete mode 100644 doc/manual/packages/sharing-packages.xml
 delete mode 100644 doc/manual/packages/ssh-substituter.xml
 delete mode 100644 doc/manual/release-notes/release-notes.xml
 delete mode 100644 doc/manual/release-notes/rl-0.10.1.xml
 delete mode 100644 doc/manual/release-notes/rl-0.10.xml
 delete mode 100644 doc/manual/release-notes/rl-0.11.xml
 delete mode 100644 doc/manual/release-notes/rl-0.12.xml
 delete mode 100644 doc/manual/release-notes/rl-0.13.xml
 delete mode 100644 doc/manual/release-notes/rl-0.14.xml
 delete mode 100644 doc/manual/release-notes/rl-0.15.xml
 delete mode 100644 doc/manual/release-notes/rl-0.16.xml
 delete mode 100644 doc/manual/release-notes/rl-0.5.xml
 delete mode 100644 doc/manual/release-notes/rl-0.6.xml
 delete mode 100644 doc/manual/release-notes/rl-0.7.xml
 delete mode 100644 doc/manual/release-notes/rl-0.8.1.xml
 delete mode 100644 doc/manual/release-notes/rl-0.8.xml
 delete mode 100644 doc/manual/release-notes/rl-0.9.1.xml
 delete mode 100644 doc/manual/release-notes/rl-0.9.2.xml
 delete mode 100644 doc/manual/release-notes/rl-0.9.xml
 delete mode 100644 doc/manual/release-notes/rl-1.0.xml
 delete mode 100644 doc/manual/release-notes/rl-1.1.xml
 delete mode 100644 doc/manual/release-notes/rl-1.10.xml
 delete mode 100644 doc/manual/release-notes/rl-1.11.10.xml
 delete mode 100644 doc/manual/release-notes/rl-1.11.xml
 delete mode 100644 doc/manual/release-notes/rl-1.2.xml
 delete mode 100644 doc/manual/release-notes/rl-1.3.xml
 delete mode 100644 doc/manual/release-notes/rl-1.4.xml
 delete mode 100644 doc/manual/release-notes/rl-1.5.1.xml
 delete mode 100644 doc/manual/release-notes/rl-1.5.2.xml
 delete mode 100644 doc/manual/release-notes/rl-1.5.xml
 delete mode 100644 doc/manual/release-notes/rl-1.6.1.xml
 delete mode 100644 doc/manual/release-notes/rl-1.6.xml
 delete mode 100644 doc/manual/release-notes/rl-1.7.xml
 delete mode 100644 doc/manual/release-notes/rl-1.8.xml
 delete mode 100644 doc/manual/release-notes/rl-1.9.xml
 delete mode 100644 doc/manual/release-notes/rl-2.0.xml
 delete mode 100644 doc/manual/release-notes/rl-2.1.xml
 delete mode 100644 doc/manual/release-notes/rl-2.2.xml
 delete mode 100644 doc/manual/release-notes/rl-2.3.xml
 delete mode 100644 doc/manual/schemas.xml

diff --git a/.gitignore b/.gitignore
index 983026570..d456d7da3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,14 +22,9 @@ perl/Makefile.config
 /corepkgs/nar/unnar.sh
 
 # /doc/manual/
-/doc/manual/manual.html
-/doc/manual/manual.xmli
-/doc/manual/manual.pdf
-/doc/manual/manual.is-valid
 /doc/manual/*.1
 /doc/manual/*.5
 /doc/manual/*.8
-/doc/manual/version.txt
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml
deleted file mode 100644
index 871b7eb1d..000000000
--- a/doc/manual/advanced-topics/advanced-topics.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<part xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      xml:id="part-advanced-topics"
-      version="5.0">
-
-<title>Advanced Topics</title>
-
-<xi:include href="distributed-builds.xml" />
-<xi:include href="cores-vs-jobs.xml" />
-<xi:include href="diff-hook.xml" />
-<xi:include href="post-build-hook.xml" />
-
-</part>
diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml
deleted file mode 100644
index a75dea37f..000000000
--- a/doc/manual/advanced-topics/cores-vs-jobs.xml
+++ /dev/null
@@ -1,121 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="chap-tuning-cores-and-jobs">
-
-<title>Tuning Cores and Jobs</title>
-
-<para>Nix has two relevant settings with regards to how your CPU cores
-will be utilized: <xref linkend="conf-cores" /> and
-<xref linkend="conf-max-jobs" />. This chapter will talk about what
-they are, how they interact, and their configuration trade-offs.</para>
-
-<variablelist>
-  <varlistentry>
-    <term><xref linkend="conf-max-jobs" /></term>
-    <listitem><para>
-      Dictates how many separate derivations will be built at the same
-      time. If you set this to zero, the local machine will do no
-      builds. Nix will still substitute from binary caches, and build
-      remotely if remote builders are configured.
-    </para></listitem>
-  </varlistentry>
-  <varlistentry>
-    <term><xref linkend="conf-cores" /></term>
-    <listitem><para>
-      Suggests how many cores each derivation should use. Similar to
-      <command>make -j</command>.
-    </para></listitem>
-  </varlistentry>
-</variablelist>
-
-<para>The <xref linkend="conf-cores" /> setting determines the value of
-<literal>NIX_BUILD_CORES</literal>. <literal>NIX_BUILD_CORES</literal> is equal
-to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
-equals <literal>0</literal>, in which case <literal>NIX_BUILD_CORES</literal>
-will be the total number of cores in the system.</para>
-
-<para>The maximum number of consumed cores is a simple multiplication,
-<xref linkend="conf-max-jobs" /> * <literal>NIX_BUILD_CORES</literal>.</para>
-
-<para>The balance on how to set these two independent variables depends
-upon each builder's workload and hardware. Here are a few example
-scenarios on a machine with 24 cores:</para>
-
-<table>
-  <caption>Balancing 24 Build Cores</caption>
-  <thead>
-    <tr>
-      <th><xref linkend="conf-max-jobs" /></th>
-      <th><xref linkend="conf-cores" /></th>
-      <th><literal>NIX_BUILD_CORES</literal></th>
-      <th>Maximum Processes</th>
-      <th>Result</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <td>1</td>
-      <td>24</td>
-      <td>24</td>
-      <td>24</td>
-      <td>
-        One derivation will be built at a time, each one can use 24
-        cores. Undersold if a job can’t use 24 cores.
-      </td>
-    </tr>
-
-    <tr>
-      <td>4</td>
-      <td>6</td>
-      <td>6</td>
-      <td>24</td>
-      <td>
-        Four derivations will be built at once, each given access to
-        six cores.
-      </td>
-    </tr>
-    <tr>
-      <td>12</td>
-      <td>6</td>
-      <td>6</td>
-      <td>72</td>
-      <td>
-        12 derivations will be built at once, each given access to six
-        cores. This configuration is over-sold. If all 12 derivations
-        being built simultaneously try to use all six cores, the
-        machine's performance will be degraded due to extensive context
-        switching between the 12 builds.
-      </td>
-    </tr>
-    <tr>
-      <td>24</td>
-      <td>1</td>
-      <td>1</td>
-      <td>24</td>
-      <td>
-        24 derivations can build at the same time, each using a single
-        core. Never oversold, but derivations which require many cores
-        will be very slow to compile.
-      </td>
-    </tr>
-    <tr>
-      <td>24</td>
-      <td>0</td>
-      <td>24</td>
-      <td>576</td>
-      <td>
-        24 derivations can build at the same time, each using all the
-        available cores of the machine. Very likely to be oversold,
-        and very likely to suffer context switches.
-      </td>
-    </tr>
-  </tbody>
-</table>
-
-<para>It is up to the derivations' build script to respect
-host's requested cores-per-build by following the value of the
-<literal>NIX_BUILD_CORES</literal> environment variable.</para>
-
-</chapter>
diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml
deleted file mode 100644
index 85cfdfc02..000000000
--- a/doc/manual/advanced-topics/diff-hook.xml
+++ /dev/null
@@ -1,202 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      xml:id="chap-diff-hook"
-      version="5.0"
-      >
-
-<title>Verifying Build Reproducibility</title>
-
-<para>Specify a program with Nix's <xref linkend="conf-diff-hook" /> to
-compare build results when two builds produce different results. Note:
-this hook is only executed if the results are not the same, this hook
-is not used for determining if the results are the same.</para>
-
-<para>For purposes of demonstration, we'll use the following Nix file,
-<filename>deterministic.nix</filename> for testing:</para>
-
-<programlisting>
-let
-  inherit (import &lt;nixpkgs&gt; {}) runCommand;
-in {
-  stable = runCommand "stable" {} ''
-    touch $out
-  '';
-
-  unstable = runCommand "unstable" {} ''
-    echo $RANDOM > $out
-  '';
-}
-</programlisting>
-
-<para>Additionally, <filename>nix.conf</filename> contains:
-
-<programlisting>
-diff-hook = /etc/nix/my-diff-hook
-run-diff-hook = true
-</programlisting>
-
-where <filename>/etc/nix/my-diff-hook</filename> is an executable
-file containing:
-
-<programlisting>
-#!/bin/sh
-exec &gt;&amp;2
-echo "For derivation $3:"
-/run/current-system/sw/bin/diff -r "$1" "$2"
-</programlisting>
-
-</para>
-
-<para>The diff hook is executed by the same user and group who ran the
-build. However, the diff hook does not have write access to the store
-path just built.</para>
-
-<section>
-  <title>
-    Spot-Checking Build Determinism
-  </title>
-
-  <para>
-    Verify a path which already exists in the Nix store by passing
-    <option>--check</option> to the build command.
-  </para>
-
-  <para>If the build passes and is deterministic, Nix will exit with a
-  status code of 0:</para>
-
-  <screen>
-$ nix-build ./deterministic.nix -A stable
-this derivation will be built:
-  /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
-building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
-/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
-
-$ nix-build ./deterministic.nix -A stable --check
-checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
-/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
-</screen>
-
-  <para>If the build is not deterministic, Nix will exit with a status
-  code of 1:</para>
-
-  <screen>
-$ nix-build ./deterministic.nix -A unstable
-this derivation will be built:
-  /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
-building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
-/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
-
-$ nix-build ./deterministic.nix -A unstable --check
-checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
-error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
-</screen>
-
-<para>In the Nix daemon's log, we will now see:
-<screen>
-For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv:
-1c1
-&lt; 8108
----
-&gt; 30204
-</screen>
-</para>
-
-  <para>Using <option>--check</option> with <option>--keep-failed</option>
-  will cause Nix to keep the second build's output in a special,
-  <literal>.check</literal> path:</para>
-
-  <screen>
-$ nix-build ./deterministic.nix -A unstable --check --keep-failed
-checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
-note: keeping build directory '/tmp/nix-build-unstable.drv-0'
-error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check'
-</screen>
-
-  <para>In particular, notice the
-  <literal>/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check</literal>
-  output. Nix has copied the build results to that directory where you
-  can examine it.</para>
-
-  <note xml:id="check-dirs-are-unregistered">
-    <title><literal>.check</literal> paths are not registered store paths</title>
-
-    <para>Check paths are not protected against garbage collection,
-    and this path will be deleted on the next garbage collection.</para>
-
-    <para>The path is guaranteed to be alive for the duration of
-    <xref linkend="conf-diff-hook" />'s execution, but may be deleted
-    any time after.</para>
-
-    <para>If the comparison is performed as part of automated tooling,
-    please use the diff-hook or author your tooling to handle the case
-    where the build was not deterministic and also a check path does
-    not exist.</para>
-  </note>
-
-  <para>
-    <option>--check</option> is only usable if the derivation has
-    been built on the system already. If the derivation has not been
-    built Nix will fail with the error:
-    <screen>
-error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible
-</screen>
-
-    Run the build without <option>--check</option>, and then try with
-    <option>--check</option> again.
-  </para>
-</section>
-
-<section>
-  <title>
-    Automatic and Optionally Enforced Determinism Verification
-  </title>
-
-  <para>
-    Automatically verify every build at build time by executing the
-    build multiple times.
-  </para>
-
-  <para>
-    Setting <xref linkend="conf-repeat" /> and
-    <xref linkend="conf-enforce-determinism" /> in your
-    <filename>nix.conf</filename> permits the automated verification
-    of every build Nix performs.
-  </para>
-
-  <para>
-    The following configuration will run each build three times, and
-    will require the build to be deterministic:
-
-    <programlisting>
-enforce-determinism = true
-repeat = 2
-</programlisting>
-  </para>
-
-  <para>
-    Setting <xref linkend="conf-enforce-determinism" /> to false as in
-    the following configuration will run the build multiple times,
-    execute the build hook, but will allow the build to succeed even
-    if it does not build reproducibly:
-
-    <programlisting>
-enforce-determinism = false
-repeat = 1
-</programlisting>
-  </para>
-
-  <para>
-    An example output of this configuration:
-    <screen>
-$ nix-build ./test.nix -A unstable
-this derivation will be built:
-  /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
-building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
-building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
-output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
-/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
-</screen>
-  </para>
-</section>
-</chapter>
diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
deleted file mode 100644
index ec9e98e77..000000000
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ /dev/null
@@ -1,190 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='chap-distributed-builds'>
-
-<title>Remote Builds</title>
-
-<para>Nix supports remote builds, where a local Nix installation can
-forward Nix builds to other machines.  This allows multiple builds to
-be performed in parallel and allows Nix to perform multi-platform
-builds in a semi-transparent way.  For instance, if you perform a
-build for a <literal>x86_64-darwin</literal> on an
-<literal>i686-linux</literal> machine, Nix can automatically forward
-the build to a <literal>x86_64-darwin</literal> machine, if
-available.</para>
-
-<para>To forward a build to a remote machine, it’s required that the
-remote machine is accessible via SSH and that it has Nix
-installed. You can test whether connecting to the remote Nix instance
-works, e.g.
-
-<screen>
-$ nix ping-store --store ssh://mac
-</screen>
-
-will try to connect to the machine named <literal>mac</literal>. It is
-possible to specify an SSH identity file as part of the remote store
-URI, e.g.
-
-<screen>
-$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key
-</screen>
-
-Since builds should be non-interactive, the key should not have a
-passphrase. Alternatively, you can load identities ahead of time into
-<command>ssh-agent</command> or <command>gpg-agent</command>.</para>
-
-<para>If you get the error
-
-<screen>
-bash: nix-store: command not found
-error: cannot connect to 'mac'
-</screen>
-
-then you need to ensure that the <literal>PATH</literal> of
-non-interactive login shells contains Nix.</para>
-
-<warning><para>If you are building via the Nix daemon, it is the Nix
-daemon user account (that is, <literal>root</literal>) that should
-have SSH access to the remote machine. If you can’t or don’t want to
-configure <literal>root</literal> to be able to access to remote
-machine, you can use a private Nix store instead by passing
-e.g. <literal>--store ~/my-nix</literal>.</para></warning>
-
-<para>The list of remote machines can be specified on the command line
-or in the Nix configuration file. The former is convenient for
-testing. For example, the following command allows you to build a
-derivation for <literal>x86_64-darwin</literal> on a Linux machine:
-
-<screen>
-$ uname
-Linux
-
-$ nix build \
-  '(with import &lt;nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
-  --builders 'ssh://mac x86_64-darwin'
-[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
-
-$ cat ./result
-Darwin
-</screen>
-
-It is possible to specify multiple builders separated by a semicolon
-or a newline, e.g.
-
-<screen>
-  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
-</screen>
-</para>
-
-<para>Each machine specification consists of the following elements,
-separated by spaces. Only the first element is required.
-To leave a field at its default, set it to <literal>-</literal>.
-
-<orderedlist>
-
-  <listitem><para>The URI of the remote store in the format
-  <literal>ssh://[<emphasis>username</emphasis>@]<emphasis>hostname</emphasis></literal>,
-  e.g. <literal>ssh://nix@mac</literal> or
-  <literal>ssh://mac</literal>. For backward compatibility,
-  <literal>ssh://</literal> may be omitted. The hostname may be an
-  alias defined in your
-  <filename>~/.ssh/config</filename>.</para></listitem>
-
-  <listitem><para>A comma-separated list of Nix platform type
-  identifiers, such as <literal>x86_64-darwin</literal>.  It is
-  possible for a machine to support multiple platform types, e.g.,
-  <literal>i686-linux,x86_64-linux</literal>. If omitted, this
-  defaults to the local platform type.</para></listitem>
-
-  <listitem><para>The SSH identity file to be used to log in to the
-  remote machine. If omitted, SSH will use its regular
-  identities.</para></listitem>
-
-  <listitem><para>The maximum number of builds that Nix will execute
-  in parallel on the machine.  Typically this should be equal to the
-  number of CPU cores.  For instance, the machine
-  <literal>itchy</literal> in the example will execute up to 8 builds
-  in parallel.</para></listitem>
-
-  <listitem><para>The “speed factor”, indicating the relative speed of
-  the machine.  If there are multiple machines of the right type, Nix
-  will prefer the fastest, taking load into account.</para></listitem>
-
-  <listitem><para>A comma-separated list of <emphasis>supported
-  features</emphasis>.  If a derivation has the
-  <varname>requiredSystemFeatures</varname> attribute, then Nix will
-  only perform the derivation on a machine that has the specified
-  features.  For instance, the attribute
-
-<programlisting>
-requiredSystemFeatures = [ "kvm" ];
-</programlisting>
-
-  will cause the build to be performed on a machine that has the
-  <literal>kvm</literal> feature.</para></listitem>
-
-  <listitem><para>A comma-separated list of <emphasis>mandatory
-  features</emphasis>.  A machine will only be used to build a
-  derivation if all of the machine’s mandatory features appear in the
-  derivation’s <varname>requiredSystemFeatures</varname>
-  attribute..</para></listitem>
-
-</orderedlist>
-
-For example, the machine specification
-
-<programlisting>
-nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
-nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
-nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        1 2 kvm benchmark
-</programlisting>
-
-specifies several machines that can perform
-<literal>i686-linux</literal> builds. However,
-<literal>poochie</literal> will only do builds that have the attribute
-
-<programlisting>
-requiredSystemFeatures = [ "benchmark" ];
-</programlisting>
-
-or
-
-<programlisting>
-requiredSystemFeatures = [ "benchmark" "kvm" ];
-</programlisting>
-
-<literal>itchy</literal> cannot do builds that require
-<literal>kvm</literal>, but <literal>scratchy</literal> does support
-such builds. For regular builds, <literal>itchy</literal> will be
-preferred over <literal>scratchy</literal> because it has a higher
-speed factor.</para>
-
-<para>Remote builders can also be configured in
-<filename>nix.conf</filename>, e.g.
-
-<programlisting>
-builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd
-</programlisting>
-
-Finally, remote builders can be configured in a separate configuration
-file included in <option>builders</option> via the syntax
-<literal>@<emphasis>file</emphasis></literal>. For example,
-
-<programlisting>
-builders = @/etc/nix/machines
-</programlisting>
-
-causes the list of machines in <filename>/etc/nix/machines</filename>
-to be included. (This is the default.)</para>
-
-<para>If you want the builders to use caches, you likely want to set
-the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
-in your local <filename>nix.conf</filename>.</para>
-
-<para>To build only on remote builders and disable building on the local machine,
-you can use the option <option>--max-jobs 0</option>.</para>
-
-</chapter>
diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
deleted file mode 100644
index 1480ab86a..000000000
--- a/doc/manual/advanced-topics/post-build-hook.xml
+++ /dev/null
@@ -1,158 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      xml:id="chap-post-build-hook"
-      version="5.0"
-      >
-
-<title>Using the <option linkend="conf-post-build-hook">post-build-hook</option></title>
-
-<section xml:id="chap-post-build-hook-caveats">
-  <title>Implementation Caveats</title>
-  <para>Here we use the post-build hook to upload to a binary cache.
-  This is a simple and working example, but it is not suitable for all
-  use cases.</para>
-
-  <para>The post build hook program runs after each executed build,
-  and blocks the build loop. The build loop exits if the hook program
-  fails.</para>
-
-  <para>Concretely, this implementation will make Nix slow or unusable
-  when the internet is slow or unreliable.</para>
-
-  <para>A more advanced implementation might pass the store paths to a
-  user-supplied daemon or queue for processing the store paths outside
-  of the build loop.</para>
-</section>
-
-<section>
-  <title>Prerequisites</title>
-
-  <para>
-    This tutorial assumes you have configured an S3-compatible binary cache
-    according to the instructions at
-    <xref linkend="ssec-s3-substituter-authenticated-writes" />, and
-    that the <literal>root</literal> user's default AWS profile can
-    upload to the bucket.
-  </para>
-</section>
-
-<section>
-  <title>Set up a Signing Key</title>
-  <para>Use <command>nix-store --generate-binary-cache-key</command> to
-  create our public and private signing keys. We will sign paths
-  with the private key, and distribute the public key for verifying
-  the authenticity of the paths.</para>
-
-  <screen>
-# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
-# cat /etc/nix/key.public
-example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
-</screen>
-
-<para>Then, add the public key and the cache URL to your
-<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" />
-and <xref linkend="conf-substituters" /> like:</para>
-
-<programlisting>
-substituters = https://cache.nixos.org/ s3://example-nix-cache
-trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
-</programlisting>
-
-<para>We will restart the Nix daemon in a later step.</para>
-</section>
-
-<section>
-  <title>Implementing the build hook</title>
-  <para>Write the following script to
-  <filename>/etc/nix/upload-to-cache.sh</filename>:
-  </para>
-
-  <programlisting>
-#!/bin/sh
-
-set -eu
-set -f # disable globbing
-export IFS=' '
-
-echo "Signing paths" $OUT_PATHS
-nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
-echo "Uploading paths" $OUT_PATHS
-exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
-</programlisting>
-
-  <note>
-    <title>Should <literal>$OUT_PATHS</literal> be quoted?</title>
-    <para>
-      The <literal>$OUT_PATHS</literal> variable is a space-separated
-      list of Nix store paths. In this case, we expect and want the
-      shell to perform word splitting to make each output path its
-      own argument to <command>nix sign-paths</command>. Nix guarantees
-      the paths will not contain any spaces, however a store path
-      might contain glob characters. The <command>set -f</command>
-      disables globbing in the shell.
-    </para>
-  </note>
-  <para>
-    Then make sure the hook program is executable by the <literal>root</literal> user:
-    <screen>
-# chmod +x /etc/nix/upload-to-cache.sh
-</screen></para>
-</section>
-
-<section>
-  <title>Updating Nix Configuration</title>
-
-  <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook,
-  by adding the following configuration snippet at the end:</para>
-
-  <programlisting>
-post-build-hook = /etc/nix/upload-to-cache.sh
-</programlisting>
-
-<para>Then, restart the <command>nix-daemon</command>.</para>
-</section>
-
-<section>
-  <title>Testing</title>
-
-  <para>Build any derivation, for example:</para>
-
-  <screen>
-$ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
-this derivation will be built:
-  /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
-building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
-running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
-post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-</screen>
-
-  <para>Then delete the path from the store, and try substituting it from the binary cache:</para>
-  <screen>
-$ rm ./result
-$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-</screen>
-
-<para>Now, copy the path back from the cache:</para>
-<screen>
-$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
-warning: you did not specify '--add-root'; the result might be removed by the garbage collector
-/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
-</screen>
-</section>
-<section>
-  <title>Conclusion</title>
-  <para>
-    We now have a Nix installation configured to automatically sign and
-    upload every local build to a remote binary cache.
-  </para>
-
-  <para>
-    Before deploying this to production, be sure to consider the
-    implementation caveats in <xref linkend="chap-post-build-hook-caveats" />.
-  </para>
-</section>
-</chapter>
diff --git a/doc/manual/command-ref/command-ref.xml b/doc/manual/command-ref/command-ref.xml
deleted file mode 100644
index cfad9b7d7..000000000
--- a/doc/manual/command-ref/command-ref.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<part xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='part-command-ref'>
-
-<title>Command Reference</title>
-
-<partintro>
-<para>This section lists commands and options that you can use when you
-work with Nix.</para>
-</partintro>
-
-<xi:include href="opt-common.xml" />
-<xi:include href="env-common.xml" />
-<xi:include href="main-commands.xml" />
-<xi:include href="utilities.xml" />
-<xi:include href="files.xml" />
-
-</part>
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
deleted file mode 100644
index 62cc117b6..000000000
--- a/doc/manual/command-ref/conf-file.xml
+++ /dev/null
@@ -1,1244 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude"
-          xml:id="sec-conf-file"
-          version="5">
-
-<refmeta>
-  <refentrytitle>nix.conf</refentrytitle>
-  <manvolnum>5</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix.conf</refname>
-  <refpurpose>Nix configuration file</refpurpose>
-</refnamediv>
-
-<refsection><title>Description</title>
-
-<para>By default Nix reads settings from the following places:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>
-      The system-wide configuration file
-      <filename><emphasis>sysconfdir</emphasis>/nix/nix.conf</filename>
-      (i.e. <filename>/etc/nix/nix.conf</filename> on most systems),
-      or <filename>$NIX_CONF_DIR/nix.conf</filename> if
-      <literal>NIX_CONF_DIR</literal> is set. Values loaded in this
-      file are not forwarded to the Nix daemon. The client assumes
-      that the daemon has already loaded them.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>
-      If <literal>NIX_USER_CONF_FILES</literal> is set, then each path separated by
-      <literal>:</literal> will be loaded in reverse order.
-    </para>
-
-    <para>
-      Otherwise it will look for <filename>nix/nix.conf</filename>
-      files in <literal>XDG_CONFIG_DIRS</literal> and
-      <literal>XDG_CONFIG_HOME</literal>.  If these are unset, it will
-      look in <filename>$HOME/.config/nix.conf</filename>.
-    </para>
-  </listitem>
-
-</itemizedlist>
-
-<para>The configuration files consist of
-<literal><emphasis>name</emphasis> =
-<emphasis>value</emphasis></literal> pairs, one per line. Other
-files can be included with a line like <literal>include
-<emphasis>path</emphasis></literal>, where
-<emphasis>path</emphasis> is interpreted relative to the current
-conf file and a missing file is an error unless
-<literal>!include</literal> is used instead.
-Comments start with a <literal>#</literal> character.  Here is an
-example configuration file:</para>
-
-<programlisting>
-keep-outputs = true       # Nice for developers
-keep-derivations = true   # Idem
-</programlisting>
-
-<para>You can override settings on the command line using the
-<option>--option</option> flag, e.g. <literal>--option keep-outputs
-false</literal>.</para>
-
-<para>The following settings are currently available:
-
-<variablelist>
-
-
-  <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term>
-
-    <listitem>
-
-      <para>A list of URI prefixes to which access is allowed in
-      restricted evaluation mode. For example, when set to
-      <literal>https://github.com/NixOS</literal>, builtin functions
-      such as <function>fetchGit</function> are allowed to access
-      <literal>https://github.com/NixOS/patchelf.git</literal>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term>
-
-    <listitem><para>By default, Nix allows you to <function>import</function> from a derivation,
-    allowing building at evaluation time. With this option set to false, Nix will throw an error
-    when evaluating an expression that uses this feature, allowing users to ensure their evaluation
-    will not require any builds to take place.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term>
-
-    <listitem><para>(Linux-specific.) By default, builders on Linux
-    cannot acquire new privileges by calling setuid/setgid programs or
-    programs that have file capabilities. For example, programs such
-    as <command>sudo</command> or <command>ping</command> will
-    fail. (Note that in sandbox builds, no such programs are available
-    unless you bind-mount them into the sandbox via the
-    <option>sandbox-paths</option> option.) You can allow the
-    use of such programs by enabling this option. This is impure and
-    usually undesirable, but may be useful in certain scenarios
-    (e.g. to spin up containers or set up userspace network interfaces
-    in tests).</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term>
-
-    <listitem>
-
-      <para>A list of names of users (separated by whitespace) that
-      are allowed to connect to the Nix daemon. As with the
-      <option>trusted-users</option> option, you can specify groups by
-      prefixing them with <literal>@</literal>. Also, you can allow
-      all users by specifying <literal>*</literal>. The default is
-      <literal>*</literal>.</para>
-
-      <para>Note that trusted users are always allowed to connect.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix
-    automatically detects files in the store that have identical
-    contents, and replaces them with hard links to a single copy.
-    This saves disk space.  If set to <literal>false</literal> (the
-    default), you can still run <command>nix-store
-    --optimise</command> to get rid of duplicate
-    files.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-builders">
-    <term><literal>builders</literal></term>
-    <listitem>
-      <para>A list of machines on which to perform builds. <phrase
-      condition="manual">See <xref linkend="chap-distributed-builds"
-      /> for details.</phrase></para>
-    </listitem>
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-builders-use-substitutes"><term><literal>builders-use-substitutes</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix will instruct
-    remote build machines to use their own binary substitutes if available. In
-    practical terms, this means that remote hosts will fetch as many build
-    dependencies as possible from their own substitutes (e.g, from
-    <literal>cache.nixos.org</literal>), instead of waiting for this host to
-    upload them all. This can drastically reduce build times if the network
-    connection between this computer and the remote build host is slow. Defaults
-    to <literal>false</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term>
-
-    <listitem><para>This options specifies the Unix group containing
-    the Nix build user accounts.  In multi-user Nix installations,
-    builds should not be performed by the Nix account since that would
-    allow users to arbitrarily modify the Nix store and database by
-    supplying specially crafted builders; and they cannot be performed
-    by the calling user since that would allow him/her to influence
-    the build result.</para>
-
-    <para>Therefore, if this option is non-empty and specifies a valid
-    group, builds will be performed under the user accounts that are a
-    member of the group specified here (as listed in
-    <filename>/etc/group</filename>).  Those user accounts should not
-    be used for any other purpose!</para>
-
-    <para>Nix will never run two builds under the same user account at
-    the same time.  This is to prevent an obvious security hole: a
-    malicious user writing a Nix expression that modifies the build
-    result of a legitimate Nix expression being built by another user.
-    Therefore it is good to have as many Nix build user accounts as
-    you can spare.  (Remember: uids are cheap.)</para>
-
-    <para>The build users should have permission to create files in
-    the Nix store, but not delete them.  Therefore,
-    <filename>/nix/store</filename> should be owned by the Nix
-    account, its group should be the group specified here, and its
-    mode should be <literal>1775</literal>.</para>
-
-    <para>If the build users group is empty, builds will be performed
-    under the uid of the Nix process (that is, the uid of the caller
-    if <literal>NIX_REMOTE</literal> is empty, the uid under which the Nix
-    daemon runs if <literal>NIX_REMOTE</literal> is
-    <literal>daemon</literal>).  Obviously, this should not be used in
-    multi-user settings with untrusted users.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-compress-build-log"><term><literal>compress-build-log</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    build logs written to <filename>/nix/var/log/nix/drvs</filename>
-    will be compressed on the fly using bzip2.  Otherwise, they will
-    not be compressed.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
-
-    <listitem>
-
-      <para>The timeout (in seconds) for establishing connections in
-      the binary cache substituter.  It corresponds to
-      <command>curl</command>’s <option>--connect-timeout</option>
-      option.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term>
-
-    <listitem><para>Sets the value of the
-    <literal>NIX_BUILD_CORES</literal> environment variable in the
-    invocation of builders.  Builders can use this variable at their
-    discretion to control the maximum amount of parallelism.  For
-    instance, in Nixpkgs, if the derivation attribute
-    <varname>enableParallelBuilding</varname> is set to
-    <literal>true</literal>, the builder passes the
-    <option>-j<emphasis>N</emphasis></option> flag to GNU Make.
-    It can be overridden using the <option
-    linkend='opt-cores'>--cores</option> command line switch and
-    defaults to <literal>1</literal>.  The value <literal>0</literal>
-    means that the builder should use all available CPU cores in the
-    system.</para>
-
-    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term>
-  <listitem>
-    <para>
-      Absolute path to an executable capable of diffing build results.
-      The hook executes if <xref linkend="conf-run-diff-hook" /> is
-      true, and the output of a build is known to not be the same.
-      This program is not executed to determine if two results are the
-      same.
-    </para>
-
-    <para>
-      The diff hook is executed by the same user and group who ran the
-      build. However, the diff hook does not have write access to the
-      store path just built.
-    </para>
-
-    <para>The diff hook program receives three parameters:</para>
-
-    <orderedlist>
-      <listitem>
-        <para>
-          A path to the previous build's results
-        </para>
-      </listitem>
-
-      <listitem>
-        <para>
-          A path to the current build's results
-        </para>
-      </listitem>
-
-      <listitem>
-        <para>
-          The path to the build's derivation
-        </para>
-      </listitem>
-
-      <listitem>
-        <para>
-          The path to the build's scratch directory. This directory
-          will exist only if the build was run with
-          <option>--keep-failed</option>.
-        </para>
-      </listitem>
-    </orderedlist>
-
-    <para>
-      The stderr and stdout output from the diff hook will not be
-      displayed to the user. Instead, it will print to the nix-daemon's
-      log.
-    </para>
-
-    <para>When using the Nix daemon, <literal>diff-hook</literal> must
-    be set in the <filename>nix.conf</filename> configuration file, and
-    cannot be passed at the command line.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-enforce-determinism">
-    <term><literal>enforce-determinism</literal></term>
-
-    <listitem><para>See <xref linkend="conf-repeat" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-extra-sandbox-paths">
-    <term><literal>extra-sandbox-paths</literal></term>
-
-    <listitem><para>A list of additional paths appended to
-    <option>sandbox-paths</option>. Useful if you want to extend
-    its default value.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-extra-platforms"><term><literal>extra-platforms</literal></term>
-
-    <listitem><para>Platforms other than the native one which
-    this machine is capable of building for. This can be useful for
-    supporting additional architectures on compatible machines:
-    i686-linux can be built on x86_64-linux machines (and the default
-    for this setting reflects this); armv7 is backwards-compatible with
-    armv6 and armv5tel; some aarch64 machines can also natively run
-    32-bit ARM code; and qemu-user may be used to support non-native
-    platforms (though this may be slow and buggy). Most values for this
-    are not enabled by default because build systems will often
-    misdetect the target platform and generate incompatible code, so you
-    may wish to cross-check the results of using this option against
-    proper natively-built versions of your
-    derivations.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term>
-
-    <listitem><para>Additional binary caches appended to those
-    specified in <option>substituters</option>.  When used by
-    unprivileged users, untrusted substituters (i.e. those not listed
-    in <option>trusted-substituters</option>) are silently
-    ignored.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix will fall
-    back to building from source if a binary substitute fails.  This
-    is equivalent to the <option>--fallback</option> flag.  The
-    default is <literal>false</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, changes to the
-    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
-    synchronously flushed to disk.  This improves robustness in case
-    of system crashes, but reduces performance.  The default is
-    <literal>true</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
-
-    <listitem><para>A list of web servers used by
-    <function>builtins.fetchurl</function> to obtain files by
-    hash. The default is
-    <literal>http://tarballs.nixos.org/</literal>. Given a hash type
-    <emphasis>ht</emphasis> and a base-16 hash
-    <emphasis>h</emphasis>, Nix will try to download the file
-    from
-    <literal>hashed-mirror/<emphasis>ht</emphasis>/<emphasis>h</emphasis></literal>.
-    This allows files to be downloaded even if they have disappeared
-    from their original URI. For example, given the default mirror
-    <literal>http://tarballs.nixos.org/</literal>, when building the derivation
-
-<programlisting>
-builtins.fetchurl {
-  url = "https://example.org/foo-1.2.3.tar.xz";
-  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-</programlisting>
-
-    Nix will attempt to download this file from
-    <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
-    first. If it is not available there, if will try the original URI.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
-
-    <listitem><para>The maximum number of parallel TCP connections
-    used to fetch files from binary caches and by other downloads. It
-    defaults to 25. 0 means no limit.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-keep-build-log"><term><literal>keep-build-log</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    Nix will write the build log of a derivation (i.e. the standard
-    output and error of its builder) to the directory
-    <filename>/nix/var/log/nix/drvs</filename>.  The build log can be
-    retrieved using the command <command>nix-store -l
-    <emphasis>path</emphasis></command>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term>
-
-    <listitem><para>If <literal>true</literal> (default), the garbage
-    collector will keep the derivations from which non-garbage store
-    paths were built.  If <literal>false</literal>, they will be
-    deleted unless explicitly registered as a root (or reachable from
-    other roots).</para>
-
-    <para>Keeping derivation around is useful for querying and
-    traceability (e.g., it allows you to ask with what dependencies or
-    options a store path was built), so by default this option is on.
-    Turn it off to save a bit of disk space (or a lot if
-    <literal>keep-outputs</literal> is also turned on).</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
-
-    <listitem><para>If <literal>false</literal> (default), derivations
-    are not stored in Nix user environments.  That is, the derivations of
-    any build-time-only dependencies may be garbage-collected.</para>
-
-    <para>If <literal>true</literal>, when you add a Nix derivation to
-    a user environment, the path of the derivation is stored in the
-    user environment.  Thus, the derivation will not be
-    garbage-collected until the user environment generation is deleted
-    (<command>nix-env --delete-generations</command>).  To prevent
-    build-time-only dependencies from being collected, you should also
-    turn on <literal>keep-outputs</literal>.</para>
-
-    <para>The difference between this option and
-    <literal>keep-derivations</literal> is that this one is
-    “sticky”: it applies to any user environment created while this
-    option was enabled, while <literal>keep-derivations</literal>
-    only applies at the moment the garbage collector is
-    run.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
-
-    <listitem><para>If <literal>true</literal>, the garbage collector
-    will keep the outputs of non-garbage derivations.  If
-    <literal>false</literal> (default), outputs will be deleted unless
-    they are GC roots themselves (or reachable from other roots).</para>
-
-    <para>In general, outputs must be registered as roots separately.
-    However, even if the output of a derivation is registered as a
-    root, the collector will still delete store paths that are used
-    only at build time (e.g., the C compiler, or source tarballs
-    downloaded from the network).  To prevent it from doing so, set
-    this option to <literal>true</literal>.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of bytes that a
-      builder can write to its stdout/stderr.  If the builder exceeds
-      this limit, it’s killed.  A value of <literal>0</literal> (the
-      default) means that there is no limit.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term>
-
-    <listitem><para>When a garbage collection is triggered by the
-    <literal>min-free</literal> option, it stops as soon as
-    <literal>max-free</literal> bytes are available. The default is
-    infinity (i.e. delete all garbage).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
-
-    <listitem><para>This option defines the maximum number of jobs
-    that Nix will try to build in parallel.  The default is
-    <literal>1</literal>. The special value <literal>auto</literal>
-    causes Nix to use the number of CPUs in your system.  <literal>0</literal>
-    is useful when using remote builders to prevent any local builds (except for
-    <literal>preferLocalBuild</literal> derivation attribute which executes locally
-    regardless).  It can be
-    overridden using the <option
-    linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
-    command line switch.</para>
-
-    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of seconds that a
-      builder can go without producing any data on standard output or
-      standard error.  This is useful (for instance in an automated
-      build system) to catch builds that are stuck in an infinite
-      loop, or to catch remote builds that are hanging due to network
-      problems.  It can be overridden using the <option
-      linkend="opt-max-silent-time">--max-silent-time</option> command
-      line switch.</para>
-
-      <para>The value <literal>0</literal> means that there is no
-      timeout.  This is also the default.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term>
-
-    <listitem>
-      <para>When free disk space in <filename>/nix/store</filename>
-      drops below <literal>min-free</literal> during a build, Nix
-      performs a garbage-collection until <literal>max-free</literal>
-      bytes are available or there is no more garbage.  A value of
-      <literal>0</literal> (the default) disables this feature.</para>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-narinfo-cache-negative-ttl"><term><literal>narinfo-cache-negative-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for negative lookups. If a store path is
-      queried from a substituter but was not found, there will be a
-      negative lookup cached in the local disk cache database for the
-      specified duration.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-narinfo-cache-positive-ttl"><term><literal>narinfo-cache-positive-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for positive lookups. If a store path is
-      queried from a substituter, the result of the query will be cached
-      in the local disk cache database including some of the NAR
-      metadata. The default TTL is a month, setting a shorter TTL for
-      positive lookups can be useful for binary caches that have
-      frequent garbage collection, in which case having a more frequent
-      cache invalidation would prevent trying to pull the path again and
-      failing with a hash mismatch if the build isn't reproducible.
-      </para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
-
-    <listitem><para>If set to an absolute path to a <filename>netrc</filename>
-    file, Nix will use the HTTP authentication credentials in this file when
-    trying to download from a remote host through HTTP or HTTPS. Defaults to
-    <filename>$NIX_CONF_DIR/netrc</filename>.</para>
-
-    <para>The <filename>netrc</filename> file consists of a list of
-    accounts in the following format:
-
-<screen>
-machine <emphasis>my-machine</emphasis>
-login <emphasis>my-username</emphasis>
-password <emphasis>my-password</emphasis>
-</screen>
-
-    For the exact syntax, see <link
-    xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
-    <literal>curl</literal> documentation.</link></para>
-
-    <note><para>This must be an absolute path, and <literal>~</literal>
-    is not resolved. For example, <filename>~/.netrc</filename> won't
-    resolve to your home directory's <filename>.netrc</filename>.</para></note>
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-plugin-files">
-    <term><literal>plugin-files</literal></term>
-    <listitem>
-      <para>
-        A list of plugin files to be loaded by Nix. Each of these
-        files will be dlopened by Nix, allowing them to affect
-        execution through static initialization. In particular, these
-        plugins may construct static instances of RegisterPrimOp to
-        add new primops or constants to the expression language,
-        RegisterStoreImplementation to add new store implementations,
-        RegisterCommand to add new subcommands to the
-        <literal>nix</literal> command, and RegisterSetting to add new
-        nix config settings. See the constructors for those types for
-        more details.
-      </para>
-      <para>
-        Since these files are loaded into the same address space as
-        Nix itself, they must be DSOs compatible with the instance of
-        Nix running at the time (i.e. compiled against the same
-        headers, not linked to any incompatible libraries). They
-        should not be linked to any Nix libs directly, as those will
-        be available already at load time.
-      </para>
-      <para>
-        If an entry in the list is a directory, all files in the
-        directory are loaded as plugins (non-recursively).
-      </para>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
-
-    <listitem>
-
-
-      <para>If set, the path to a program that can set extra
-      derivation-specific settings for this system. This is used for settings
-      that can't be captured by the derivation model itself and are too variable
-      between different versions of the same system to be hard-coded into nix.
-      </para>
-
-      <para>The hook is passed the derivation path and, if sandboxes are enabled,
-      the sandbox directory. It can then modify the sandbox and send a series of
-      commands to modify various settings to stdout. The currently recognized
-      commands are:</para>
-
-      <variablelist>
-        <varlistentry xml:id="extra-sandbox-paths">
-          <term><literal>extra-sandbox-paths</literal></term>
-
-          <listitem>
-
-            <para>Pass a list of files and directories to be included in the
-            sandbox for this build. One entry per line, terminated by an empty
-            line. Entries have the same format as
-            <literal>sandbox-paths</literal>.</para>
-
-          </listitem>
-
-        </varlistentry>
-      </variablelist>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-post-build-hook">
-    <term><literal>post-build-hook</literal></term>
-    <listitem>
-      <para>Optional. The path to a program to execute after each build.</para>
-
-      <para>This option is only settable in the global
-      <filename>nix.conf</filename>, or on the command line by trusted
-      users.</para>
-
-      <para>When using the nix-daemon, the daemon executes the hook as
-      <literal>root</literal>. If the nix-daemon is not involved, the
-      hook runs as the user executing the nix-build.</para>
-
-      <itemizedlist>
-        <listitem><para>The hook executes after an evaluation-time build.</para></listitem>
-        <listitem><para>The hook does not execute on substituted paths.</para></listitem>
-        <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
-        <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
-        <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem>
-      </itemizedlist>
-
-      <para>The program executes with no arguments. The program's environment
-      contains the following environment variables:</para>
-
-      <variablelist>
-        <varlistentry>
-          <term><literal>DRV_PATH</literal></term>
-          <listitem>
-            <para>The derivation for the built paths.</para>
-            <para>Example:
-            <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal>
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><literal>OUT_PATHS</literal></term>
-          <listitem>
-            <para>Output paths of the built derivation, separated by a space character.</para>
-            <para>Example:
-            <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
-            /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
-            /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
-            /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
-            /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-      </variablelist>
-
-      <para>See <xref linkend="chap-post-build-hook" /> for an example
-      implementation.</para>
-
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
-
-    <listitem><para>How many times to repeat builds to check whether
-    they are deterministic. The default value is 0. If the value is
-    non-zero, every build is repeated the specified number of
-    times. If the contents of any of the runs differs from the
-    previous ones and <xref linkend="conf-enforce-determinism" /> is
-    true, the build is rejected and the resulting store paths are not
-    registered as “valid” in Nix’s database.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    any non-content-addressed path added or copied to the Nix store
-    (e.g. when substituting from a binary cache) must have a valid
-    signature, that is, be signed using one of the keys listed in
-    <option>trusted-public-keys</option> or
-    <option>secret-key-files</option>. Set to <literal>false</literal>
-    to disable signature checking.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-restrict-eval"><term><literal>restrict-eval</literal></term>
-
-    <listitem>
-
-      <para>If set to <literal>true</literal>, the Nix evaluator will
-      not allow access to any files outside of the Nix search path (as
-      set via the <literal>NIX_PATH</literal> environment variable or the
-      <option>-I</option> option), or to URIs outside of
-      <option>allowed-uri</option>. The default is
-      <literal>false</literal>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-run-diff-hook"><term><literal>run-diff-hook</literal></term>
-  <listitem>
-    <para>
-      If true, enable the execution of <xref linkend="conf-diff-hook" />.
-    </para>
-
-    <para>
-      When using the Nix daemon, <literal>run-diff-hook</literal> must
-      be set in the <filename>nix.conf</filename> configuration file,
-      and cannot be passed at the command line.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, builds will be
-    performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
-    they’re isolated from the normal file system hierarchy and will
-    only see their dependencies in the Nix store, the temporary build
-    directory, private versions of <filename>/proc</filename>,
-    <filename>/dev</filename>, <filename>/dev/shm</filename> and
-    <filename>/dev/pts</filename> (on Linux), and the paths configured with the
-    <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal>
-    option</link>. This is useful to prevent undeclared dependencies
-    on files in directories such as <filename>/usr/bin</filename>. In
-    addition, on Linux, builds run in private PID, mount, network, IPC
-    and UTS namespaces to isolate them from other processes in the
-    system (except that fixed-output derivations do not run in private
-    network namespace to ensure they can access the network).</para>
-
-    <para>Currently, sandboxing only work on Linux and macOS. The use
-    of a sandbox requires that Nix is run as root (so you should use
-    the <link linkend='conf-build-users-group'>“build users”
-    feature</link> to perform the actual builds under different users
-    than root).</para>
-
-    <para>If this option is set to <literal>relaxed</literal>, then
-    fixed-output derivations and derivations that have the
-    <varname>__noChroot</varname> attribute set to
-    <literal>true</literal> do not run in sandboxes.</para>
-
-    <para>The default is <literal>true</literal> on Linux and
-    <literal>false</literal> on all other platforms.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term>
-
-    <listitem><para>This option determines the maximum size of the
-    <literal>tmpfs</literal> filesystem mounted on
-    <filename>/dev/shm</filename> in Linux sandboxes. For the format,
-    see the description of the <option>size</option> option of
-    <literal>tmpfs</literal> in
-    <citerefentry><refentrytitle>mount</refentrytitle><manvolnum>8</manvolnum></citerefentry>. The
-    default is <literal>50%</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-sandbox-paths">
-    <term><literal>sandbox-paths</literal></term>
-
-    <listitem><para>A list of paths bind-mounted into Nix sandbox
-    environments. You can use the syntax
-    <literal><emphasis>target</emphasis>=<emphasis>source</emphasis></literal>
-    to mount a path in a different location in the sandbox; for
-    instance, <literal>/bin=/nix-bin</literal> will mount the path
-    <literal>/nix-bin</literal> as <literal>/bin</literal> inside the
-    sandbox. If <emphasis>source</emphasis> is followed by
-    <literal>?</literal>, then it is not an error if
-    <emphasis>source</emphasis> does not exist; for example,
-    <literal>/dev/nvidiactl?</literal> specifies that
-    <filename>/dev/nvidiactl</filename> will only be mounted in the
-    sandbox if it exists in the host filesystem.</para>
-
-    <para>Depending on how Nix was built, the default value for this option
-    may be empty or provide <filename>/bin/sh</filename> as a
-    bind-mount of <command>bash</command>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-secret-key-files"><term><literal>secret-key-files</literal></term>
-
-    <listitem><para>A whitespace-separated list of files containing
-    secret (private) keys. These are used to sign locally-built
-    paths. They can be generated using <command>nix-store
-    --generate-binary-cache-key</command>. The corresponding public
-    key can be distributed to other users, who can add it to
-    <option>trusted-public-keys</option> in their
-    <filename>nix.conf</filename>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term>
-
-    <listitem><para>Causes Nix to print out a stack trace in case of Nix
-    expression evaluation errors.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-substitute"><term><literal>substitute</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (default), Nix
-    will use binary substitutes if available.  This option can be
-    disabled to force building from source.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term>
-    <listitem>
-      <para>The timeout (in seconds) for receiving data from servers
-      during download. Nix cancels idle downloads after this timeout's
-      duration.</para>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
-
-    <listitem><para>A list of URLs of substituters, separated by
-    whitespace.  The default is
-    <literal>https://cache.nixos.org</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-system"><term><literal>system</literal></term>
-
-    <listitem><para>This option specifies the canonical Nix system
-    name of the current installation, such as
-    <literal>i686-linux</literal> or
-    <literal>x86_64-darwin</literal>.  Nix can only build derivations
-    whose <literal>system</literal> attribute equals the value
-    specified here.  In general, it never makes sense to modify this
-    value from its default, since you can use it to ‘lie’ about the
-    platform you are building on (e.g., perform a Mac OS build on a
-    Linux machine; the result would obviously be wrong).  It only
-    makes sense if the Nix binaries can run on multiple platforms,
-    e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and
-    <literal>i686-linux</literal>.</para>
-
-    <para>It defaults to the canonical Nix system name detected by
-    <filename>configure</filename> at build time.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term>
-
-    <listitem><para>A set of system “features” supported by this
-    machine, e.g. <literal>kvm</literal>. Derivations can express a
-    dependency on such features through the derivation attribute
-    <varname>requiredSystemFeatures</varname>. For example, the
-    attribute
-
-<programlisting>
-requiredSystemFeatures = [ "kvm" ];
-</programlisting>
-
-    ensures that the derivation can only be built on a machine with
-    the <literal>kvm</literal> feature.</para>
-
-    <para>This setting by default includes <literal>kvm</literal> if
-    <filename>/dev/kvm</filename> is accessible, and the
-    pseudo-features <literal>nixos-test</literal>,
-    <literal>benchmark</literal> and <literal>big-parallel</literal>
-    that are used in Nixpkgs to route builds to specific
-    machines.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-tarball-ttl"><term><literal>tarball-ttl</literal></term>
-
-    <listitem>
-      <para>Default: <literal>3600</literal> seconds.</para>
-
-      <para>The number of seconds a downloaded tarball is considered
-      fresh. If the cached tarball is stale, Nix will check whether
-      it is still up to date using the ETag header. Nix will download
-      a new version if the ETag header is unsupported, or the
-      cached ETag doesn't match.
-      </para>
-
-      <para>Setting the TTL to <literal>0</literal> forces Nix to always
-      check if the tarball is up to date.</para>
-
-      <para>Nix caches tarballs in
-      <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para>
-
-      <para>Files fetched via <literal>NIX_PATH</literal>,
-      <function>fetchGit</function>, <function>fetchMercurial</function>,
-      <function>fetchTarball</function>, and <function>fetchurl</function>
-      respect this TTL.
-      </para>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of seconds that a
-      builder can run.  This is useful (for instance in an automated
-      build system) to catch builds that are stuck in an infinite loop
-      but keep writing to their standard output or standard error.  It
-      can be overridden using the <option
-      linkend="opt-timeout">--timeout</option> command line
-      switch.</para>
-
-      <para>The value <literal>0</literal> means that there is no
-      timeout.  This is also the default.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term>
-
-    <listitem>
-
-      <para>Default: <literal>false</literal>.</para>
-
-      <para>If set to <literal>true</literal>, the Nix evaluator will
-      trace every function call. Nix will print a log message at the
-      "vomit" level for every function entrance and function exit.</para>
-
-      <informalexample><screen>
-function-trace entered undefined position at 1565795816999559622
-function-trace exited undefined position at 1565795816999581277
-function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
-function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
-</screen></informalexample>
-
-      <para>The <literal>undefined position</literal> means the function
-      call is a builtin.</para>
-
-      <para>Use the <literal>contrib/stack-collapse.py</literal> script
-      distributed with the Nix source code to convert the trace logs
-      in to a format suitable for <command>flamegraph.pl</command>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
-
-    <listitem><para>A whitespace-separated list of public keys. When
-    paths are copied from another Nix store (such as a binary cache),
-    they must be signed with one of these keys. For example:
-    <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term>
-
-    <listitem><para>A list of URLs of substituters, separated by
-    whitespace.  These are not used by default, but can be enabled by
-    users of the Nix daemon by specifying <literal>--option
-    substituters <emphasis>urls</emphasis></literal> on the
-    command line.  Unprivileged users are only allowed to pass a
-    subset of the URLs listed in <literal>substituters</literal> and
-    <literal>trusted-substituters</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
-
-    <listitem>
-
-      <para>A list of names of users (separated by whitespace) that
-      have additional rights when connecting to the Nix daemon, such
-      as the ability to specify additional binary caches, or to import
-      unsigned NARs. You can also specify groups by prefixing them
-      with <literal>@</literal>; for instance,
-      <literal>@wheel</literal> means all users in the
-      <literal>wheel</literal> group. The default is
-      <literal>root</literal>.</para>
-
-      <warning><para>Adding a user to <option>trusted-users</option>
-      is essentially equivalent to giving that user root access to the
-      system. For example, the user can set
-      <option>sandbox-paths</option> and thereby obtain read access to
-      directories that are otherwise inacessible to
-      them.</para></warning>
-
-    </listitem>
-
-  </varlistentry>
-
-</variablelist>
-</para>
-
-<refsection>
-  <title>Deprecated Settings</title>
-
-<para>
-
-<variablelist>
-
-  <varlistentry xml:id="conf-binary-caches">
-    <term><literal>binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>binary-caches</literal> is now an alias to
-    <xref linkend="conf-substituters" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-binary-cache-public-keys">
-    <term><literal>binary-cache-public-keys</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>binary-cache-public-keys</literal> is now an alias to
-    <xref linkend="conf-trusted-public-keys" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-compress-log">
-    <term><literal>build-compress-log</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-compress-log</literal> is now an alias to
-    <xref linkend="conf-compress-build-log" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-cores">
-    <term><literal>build-cores</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-cores</literal> is now an alias to
-    <xref linkend="conf-cores" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-extra-chroot-dirs">
-    <term><literal>build-extra-chroot-dirs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-extra-chroot-dirs</literal> is now an alias to
-    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-extra-sandbox-paths">
-    <term><literal>build-extra-sandbox-paths</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-extra-sandbox-paths</literal> is now an alias to
-    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-fallback">
-    <term><literal>build-fallback</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-fallback</literal> is now an alias to
-    <xref linkend="conf-fallback" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-jobs">
-    <term><literal>build-max-jobs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-jobs</literal> is now an alias to
-    <xref linkend="conf-max-jobs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-log-size">
-    <term><literal>build-max-log-size</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-log-size</literal> is now an alias to
-    <xref linkend="conf-max-build-log-size" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-silent-time">
-    <term><literal>build-max-silent-time</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-silent-time</literal> is now an alias to
-    <xref linkend="conf-max-silent-time" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-repeat">
-    <term><literal>build-repeat</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-repeat</literal> is now an alias to
-    <xref linkend="conf-repeat" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-timeout">
-    <term><literal>build-timeout</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-timeout</literal> is now an alias to
-    <xref linkend="conf-timeout" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-chroot">
-    <term><literal>build-use-chroot</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-chroot</literal> is now an alias to
-    <xref linkend="conf-sandbox" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-sandbox">
-    <term><literal>build-use-sandbox</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-sandbox</literal> is now an alias to
-    <xref linkend="conf-sandbox" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-substitutes">
-    <term><literal>build-use-substitutes</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-substitutes</literal> is now an alias to
-    <xref linkend="conf-substitute" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-gc-keep-derivations">
-    <term><literal>gc-keep-derivations</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>gc-keep-derivations</literal> is now an alias to
-    <xref linkend="conf-keep-derivations" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-gc-keep-outputs">
-    <term><literal>gc-keep-outputs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>gc-keep-outputs</literal> is now an alias to
-    <xref linkend="conf-keep-outputs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-env-keep-derivations">
-    <term><literal>env-keep-derivations</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>env-keep-derivations</literal> is now an alias to
-    <xref linkend="conf-keep-env-derivations" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-extra-binary-caches">
-    <term><literal>extra-binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>extra-binary-caches</literal> is now an alias to
-    <xref linkend="conf-extra-substituters" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-binary-caches">
-    <term><literal>trusted-binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>trusted-binary-caches</literal> is now an alias to
-    <xref linkend="conf-trusted-substituters" />.</para></listitem>
-  </varlistentry>
-</variablelist>
-</para>
-</refsection>
-
-</refsection>
-
-</refentry>
diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
deleted file mode 100644
index ac40fccf7..000000000
--- a/doc/manual/command-ref/env-common.xml
+++ /dev/null
@@ -1,209 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-common-env">
-
-<title>Common Environment Variables</title>
-
-
-<para>Most Nix commands interpret the following environment variables:</para>
-
-<variablelist xml:id="env-common">
-
-<varlistentry><term><literal>IN_NIX_SHELL</literal></term>
-
-  <listitem><para>Indicator that tells if the current environment was set up by
-  <command>nix-shell</command>.  Since Nix 2.0 the values are
-  <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem>
-
-</varlistentry>
-
-<varlistentry xml:id="env-NIX_PATH"><term><literal>NIX_PATH</literal></term>
-
-  <listitem>
-
-    <para>A colon-separated list of directories used to look up Nix
-    expressions enclosed in angle brackets (i.e.,
-    <literal>&lt;<emphasis>path</emphasis>></literal>).  For
-    instance, the value
-
-    <screen>
-/home/eelco/Dev:/etc/nixos</screen>
-
-    will cause Nix to look for paths relative to
-    <filename>/home/eelco/Dev</filename> and
-    <filename>/etc/nixos</filename>, in this order.  It is also
-    possible to match paths against a prefix.  For example, the value
-
-    <screen>
-nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
-
-    will cause Nix to search for
-    <literal>&lt;nixpkgs/<emphasis>path</emphasis>></literal> in
-    <filename>/home/eelco/Dev/nixpkgs-branch/<emphasis>path</emphasis></filename>
-    and
-    <filename>/etc/nixos/nixpkgs/<emphasis>path</emphasis></filename>.</para>
-
-    <para>If a path in the Nix search path starts with
-    <literal>http://</literal> or <literal>https://</literal>, it is
-    interpreted as the URL of a tarball that will be downloaded and
-    unpacked to a temporary location. The tarball must consist of a
-    single top-level directory. For example, setting
-    <literal>NIX_PATH</literal> to
-
-    <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
-
-    tells Nix to download the latest revision in the Nixpkgs/NixOS
-    15.09 channel.</para>
-
-    <para>A following shorthand can be used to refer to the official channels:
-
-    <screen>nixpkgs=channel:nixos-15.09</screen>
-    </para>
-
-    <para>The search path can be extended using the <option
-    linkend="opt-I">-I</option> option, which takes precedence over
-    <literal>NIX_PATH</literal>.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_IGNORE_SYMLINK_STORE</literal></term>
-
-  <listitem>
-
-  <para>Normally, the Nix store directory (typically
-  <filename>/nix/store</filename>) is not allowed to contain any
-  symlink components.  This is to prevent “impure” builds.  Builders
-  sometimes “canonicalise” paths by resolving all symlink components.
-  Thus, builds on different machines (with
-  <filename>/nix/store</filename> resolving to different locations)
-  could yield different results.  This is generally not a problem,
-  except when builds are deployed to machines where
-  <filename>/nix/store</filename> resolves differently.  If you are
-  sure that you’re not going to do that, you can set
-  <literal>NIX_IGNORE_SYMLINK_STORE</literal> to <literal>1</literal>.</para>
-
-  <para>Note that if you’re symlinking the Nix store so that you can
-  put it on another file system than the root file system, on Linux
-  you’re better off using <literal>bind</literal> mount points, e.g.,
-
-  <screen>
-$ mkdir /nix
-$ mount -o bind /mnt/otherdisk/nix /nix</screen>
-
-  Consult the <citerefentry><refentrytitle>mount</refentrytitle>
-  <manvolnum>8</manvolnum></citerefentry> manual page for details.</para>
-
-  </listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_STORE_DIR</literal></term>
-
-  <listitem><para>Overrides the location of the Nix store (default
-  <filename><emphasis>prefix</emphasis>/store</filename>).</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_DATA_DIR</literal></term>
-
-  <listitem><para>Overrides the location of the Nix static data
-  directory (default
-  <filename><emphasis>prefix</emphasis>/share</filename>).</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_LOG_DIR</literal></term>
-
-  <listitem><para>Overrides the location of the Nix log directory
-  (default <filename><emphasis>prefix</emphasis>/var/log/nix</filename>).</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_STATE_DIR</literal></term>
-
-  <listitem><para>Overrides the location of the Nix state directory
-  (default <filename><emphasis>prefix</emphasis>/var/nix</filename>).</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_CONF_DIR</literal></term>
-
-  <listitem><para>Overrides the location of the system Nix configuration
-  directory (default
-  <filename><emphasis>prefix</emphasis>/etc/nix</filename>).</para></listitem>
-
-</varlistentry>
-
-<varlistentry><term><literal>NIX_USER_CONF_FILES</literal></term>
-
-  <listitem><para>Overrides the location of the user Nix configuration files
-  to load from (defaults to the XDG spec locations). The variable is treated
-  as a list separated by the <literal>:</literal> token.</para></listitem>
-
-</varlistentry>
-
-<varlistentry><term><literal>TMPDIR</literal></term>
-
-  <listitem><para>Use the specified directory to store temporary
-  files.  In particular, this includes temporary build directories;
-  these can take up substantial amounts of disk space.  The default is
-  <filename>/tmp</filename>.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="envar-remote"><term><literal>NIX_REMOTE</literal></term>
-
-  <listitem><para>This variable should be set to
-  <literal>daemon</literal> if you want to use the Nix daemon to
-  execute Nix operations. This is necessary in <link
-  linkend="ssec-multi-user">multi-user Nix installations</link>.
-  If the Nix daemon's Unix socket is at some non-standard path,
-  this variable should be set to <literal>unix://path/to/socket</literal>.
-  Otherwise, it should be left unset.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_SHOW_STATS</literal></term>
-
-  <listitem><para>If set to <literal>1</literal>, Nix will print some
-  evaluation statistics, such as the number of values
-  allocated.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>NIX_COUNT_CALLS</literal></term>
-
-  <listitem><para>If set to <literal>1</literal>, Nix will print how
-  often functions were called during Nix expression evaluation.  This
-  is useful for profiling your Nix expressions.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><literal>GC_INITIAL_HEAP_SIZE</literal></term>
-
-  <listitem><para>If Nix has been configured to use the Boehm garbage
-  collector, this variable sets the initial size of the heap in bytes.
-  It defaults to 384 MiB.  Setting it to a low value reduces memory
-  consumption, but will increase runtime due to the overhead of
-  garbage collection.</para></listitem>
-
-</varlistentry>
-
-
-</variablelist>
-
-
-</chapter>
diff --git a/doc/manual/command-ref/files.xml b/doc/manual/command-ref/files.xml
deleted file mode 100644
index 7bbc96e89..000000000
--- a/doc/manual/command-ref/files.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='ch-files'>
-
-<title>Files</title>
-
-<para>This section lists configuration files that you can use when you
-work with Nix.</para>
-
-<xi:include href="conf-file.xml" />
-
-</chapter>
\ No newline at end of file
diff --git a/doc/manual/command-ref/main-commands.xml b/doc/manual/command-ref/main-commands.xml
deleted file mode 100644
index 0f4169243..000000000
--- a/doc/manual/command-ref/main-commands.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='ch-main-commands'>
-
-<title>Main Commands</title>
-
-<para>This section lists commands and options that you can use when you
-work with Nix.</para>
-
-<xi:include href="nix-env.xml" />
-<xi:include href="nix-build.xml" />
-<xi:include href="nix-shell.xml" />
-<xi:include href="nix-store.xml" />
-
-</chapter>
\ No newline at end of file
diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml
deleted file mode 100644
index ec9145143..000000000
--- a/doc/manual/command-ref/nix-build.xml
+++ /dev/null
@@ -1,190 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-build">
-
-<refmeta>
-  <refentrytitle>nix-build</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-build</refname>
-  <refpurpose>build a Nix expression</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-build</command>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg>
-      <group choice='req'>
-        <arg choice='plain'><option>--attr</option></arg>
-        <arg choice='plain'><option>-A</option></arg>
-      </group>
-      <emphasis>attrPath</emphasis>
-    </arg>
-    <arg><option>--no-out-link</option></arg>
-    <arg><option>--dry-run</option></arg>
-    <arg>
-      <group choice='req'>
-        <arg choice='plain'><option>--out-link</option></arg>
-        <arg choice='plain'><option>-o</option></arg>
-      </group>
-      <emphasis>outlink</emphasis>
-    </arg>
-    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-<refsection><title>Description</title>
-
-<para>The <command>nix-build</command> command builds the derivations
-described by the Nix expressions in <emphasis>paths</emphasis>.
-If the build succeeds, it places a symlink to the result in the
-current directory.  The symlink is called <filename>result</filename>.
-If there are multiple Nix expressions, or the Nix expressions evaluate
-to multiple derivations, multiple sequentially numbered symlinks are
-created (<filename>result</filename>, <filename>result-2</filename>,
-and so on).</para>
-
-<para>If no <emphasis>paths</emphasis> are specified, then
-<command>nix-build</command> will use <filename>default.nix</filename>
-in the current directory, if it exists.</para>
-
-<para>If an element of <emphasis>paths</emphasis> starts with
-<literal>http://</literal> or <literal>https://</literal>, it is
-interpreted as the URL of a tarball that will be downloaded and
-unpacked to a temporary location. The tarball must include a single
-top-level directory containing at least a file named
-<filename>default.nix</filename>.</para>
-
-<para><command>nix-build</command> is essentially a wrapper around
-<link
-linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
-(to translate a high-level Nix expression to a low-level store
-derivation) and <link
-linkend="rsec-nix-store-realise"><command>nix-store
---realise</command></link> (to build the store derivation).</para>
-
-<warning><para>The result of the build is automatically registered as
-a root of the Nix garbage collector.  This root disappears
-automatically when the <filename>result</filename> symlink is deleted
-or renamed.  So don’t rename the symlink.</para></warning>
-
-</refsection>
-
-
-<refsection><title>Options</title>
-
-<para>All options not listed here are passed to <command>nix-store
---realise</command>, except for <option>--arg</option> and
-<option>--attr</option> / <option>-A</option> which are passed to
-<command>nix-instantiate</command>.  <phrase condition="manual">See
-also <xref linkend="sec-common-options" />.</phrase></para>
-
-<variablelist>
-
-  <varlistentry><term><option>--no-out-link</option></term>
-
-    <listitem><para>Do not create a symlink to the output path.  Note
-    that as a result the output does not become a root of the garbage
-    collector, and so might be deleted by <command>nix-store
-    --gc</command>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--dry-run</option></term>
-   <listitem><para>Show what store paths would be built or downloaded.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /
-  <option>-o</option> <emphasis>outlink</emphasis></term>
-
-    <listitem><para>Change the name of the symlink to the output path
-    created from <filename>result</filename> to
-    <emphasis>outlink</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<para>The following common options are supported:</para>
-
-<variablelist condition="manpage">
-  <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-build '&lt;nixpkgs>' -A firefox
-store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
-/nix/store/d18hyl92g30l...-firefox-1.5.0.7
-
-$ ls -l result
-lrwxrwxrwx  <emphasis>...</emphasis>  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
-
-$ ls ./result/bin/
-firefox  firefox-config</screen>
-
-<para>If a derivation has multiple outputs,
-<command>nix-build</command> will build the default (first) output.
-You can also build all outputs:
-<screen>
-$ nix-build '&lt;nixpkgs>' -A openssl.all
-</screen>
-This will create a symlink for each output named
-<filename>result-<emphasis>outputname</emphasis></filename>.
-The suffix is omitted if the output name is <literal>out</literal>.
-So if <literal>openssl</literal> has outputs <literal>out</literal>,
-<literal>bin</literal> and <literal>man</literal>,
-<command>nix-build</command> will create symlinks
-<literal>result</literal>, <literal>result-bin</literal> and
-<literal>result-man</literal>.  It’s also possible to build a specific
-output:
-<screen>
-$ nix-build '&lt;nixpkgs>' -A openssl.man
-</screen>
-This will create a symlink <literal>result-man</literal>.</para>
-
-<para>Build a Nix expression given on the command line:
-
-<screen>
-$ nix-build -E 'with import &lt;nixpkgs> { }; runCommand "foo" { } "echo bar > $out"'
-$ cat ./result
-bar
-</screen>
-
-</para>
-
-<para>Build the GNU Hello package from the latest revision of the
-master branch of Nixpkgs:
-
-<screen>
-$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
-</screen>
-
-</para>
-
-</refsection>
-
-
-<refsection condition="manpage"><title>Environment variables</title>
-
-<variablelist>
-  <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml
deleted file mode 100644
index 2abeca0a0..000000000
--- a/doc/manual/command-ref/nix-channel.xml
+++ /dev/null
@@ -1,181 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-channel">
-
-<refmeta>
-  <refentrytitle>nix-channel</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-channel</refname>
-  <refpurpose>manage Nix channels</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-channel</command>
-    <group choice='req'>
-      <arg choice='plain'><option>--add</option> <emphasis>url</emphasis> <arg choice='opt'><emphasis>name</emphasis></arg></arg>
-      <arg choice='plain'><option>--remove</option> <emphasis>name</emphasis></arg>
-      <arg choice='plain'><option>--list</option></arg>
-      <arg choice='plain'><option>--update</option> <arg rep='repeat'><emphasis>names</emphasis></arg></arg>
-      <arg choice='plain'><option>--rollback</option> <arg choice='opt'><emphasis>generation</emphasis></arg></arg>
-    </group>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-<refsection><title>Description</title>
-
-<para>A Nix channel is a mechanism that allows you to automatically
-stay up-to-date with a set of pre-built Nix expressions.  A Nix
-channel is just a URL that points to a place containing a set of Nix
-expressions.  <phrase condition="manual">See also <xref
-linkend="sec-channels" />.</phrase></para>
-      
-<para>To see the list of official NixOS channels, visit <link
-xlink:href="https://nixos.org/channels" />.</para>
-
-<para>This command has the following operations:
-
-<variablelist>
-
-  <varlistentry><term><option>--add</option> <emphasis>url</emphasis> [<emphasis>name</emphasis>]</term>
-
-    <listitem><para>Adds a channel named
-    <emphasis>name</emphasis> with URL
-    <emphasis>url</emphasis> to the list of subscribed channels.
-    If <emphasis>name</emphasis> is omitted, it defaults to the
-    last component of <emphasis>url</emphasis>, with the
-    suffixes <literal>-stable</literal> or
-    <literal>-unstable</literal> removed.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--remove</option> <emphasis>name</emphasis></term>
-
-    <listitem><para>Removes the channel named
-    <emphasis>name</emphasis> from the list of subscribed
-    channels.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--list</option></term>
-
-    <listitem><para>Prints the names and URLs of all subscribed
-    channels on standard output.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--update</option> [<emphasis>names</emphasis>…]</term>
-
-    <listitem><para>Downloads the Nix expressions of all subscribed
-    channels (or only those included in
-    <emphasis>names</emphasis> if specified) and makes them the
-    default for <command>nix-env</command> operations (by symlinking
-    them from the directory
-    <filename>~/.nix-defexpr</filename>).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--rollback</option> [<emphasis>generation</emphasis>]</term>
-
-    <listitem><para>Reverts the previous call to <command>nix-channel
-    --update</command>. Optionally, you can specify a specific channel
-    generation number to restore.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</para>
-
-<para>Note that <option>--add</option> does not automatically perform
-an update.</para>
-
-<para>The list of subscribed channels is stored in
-<filename>~/.nix-channels</filename>.</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<para>To subscribe to the Nixpkgs channel and install the GNU Hello package:</para>
-
-<screen>
-$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-$ nix-channel --update
-$ nix-env -iA nixpkgs.hello</screen>
-
-<para>You can revert channel updates using <option>--rollback</option>:</para>
-
-<screen>
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
-"14.04.527.0e935f1"
-
-$ nix-channel --rollback
-switching from generation 483 to 482
-
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
-"14.04.526.dbadfad"
-</screen>
-
-</refsection>
-
-<refsection><title>Files</title>
-
-<variablelist>
-
-  <varlistentry><term><filename>/nix/var/nix/profiles/per-user/<emphasis>username</emphasis>/channels</filename></term>
-
-    <listitem><para><command>nix-channel</command> uses a
-    <command>nix-env</command> profile to keep track of previous
-    versions of the subscribed channels. Every time you run
-    <command>nix-channel --update</command>, a new channel generation
-    (that is, a symlink to the channel Nix expressions in the Nix store)
-    is created. This enables <command>nix-channel --rollback</command>
-    to revert to previous versions.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><filename>~/.nix-defexpr/channels</filename></term>
-
-    <listitem><para>This is a symlink to
-    <filename>/nix/var/nix/profiles/per-user/<emphasis>username</emphasis>/channels</filename>. It
-    ensures that <command>nix-env</command> can find your channels. In
-    a multi-user installation, you may also have
-    <filename>~/.nix-defexpr/channels_root</filename>, which links to
-    the channels of the root user.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-<refsection><title>Channel format</title>
-
-<para>A channel URL should point to a directory containing the
-following files:</para>
-
-<variablelist>
-
-  <varlistentry><term><filename>nixexprs.tar.xz</filename></term>
-
-    <listitem><para>A tarball containing Nix expressions and files
-    referenced by them (such as build scripts and patches). At the
-    top level, the tarball should contain a single directory. That
-    directory must contain a file <filename>default.nix</filename>
-    that serves as the channel’s “entry point”.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-collect-garbage.xml b/doc/manual/command-ref/nix-collect-garbage.xml
deleted file mode 100644
index cbcb5add5..000000000
--- a/doc/manual/command-ref/nix-collect-garbage.xml
+++ /dev/null
@@ -1,63 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-collect-garbage">
-  
-<refmeta>
-  <refentrytitle>nix-collect-garbage</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-collect-garbage</refname>
-  <refpurpose>delete unreachable store paths</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-collect-garbage</command>
-    <arg><option>--delete-old</option></arg>
-    <arg><option>-d</option></arg>
-    <arg><option>--delete-older-than</option> <emphasis>period</emphasis></arg>
-    <arg><option>--max-freed</option> <emphasis>bytes</emphasis></arg>
-    <arg><option>--dry-run</option></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-collect-garbage</command> is mostly an
-alias of <link linkend="rsec-nix-store-gc"><command>nix-store
---gc</command></link>, that is, it deletes all unreachable paths in
-the Nix store to clean up your system.  However, it provides two
-additional options: <option>-d</option> (<option>--delete-old</option>),
-which deletes all old generations of all profiles in
-<filename>/nix/var/nix/profiles</filename> by invoking
-<literal>nix-env --delete-generations old</literal> on all profiles
-(of course, this makes rollbacks to previous configurations
-impossible); and
-<option>--delete-older-than</option> <emphasis>period</emphasis>,
-where period is a value such as <literal>30d</literal>, which deletes
-all generations older than the specified number of days in all profiles
-in <filename>/nix/var/nix/profiles</filename> (except for the generations
-that were active at that point in time).
-</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<para>To delete from the Nix store everything that is not used by the
-current generations of each profile, do
-
-<screen>
-$ nix-collect-garbage -d</screen>
-
-</para>
-
-</refsection>
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml
deleted file mode 100644
index 6fc8c312b..000000000
--- a/doc/manual/command-ref/nix-copy-closure.xml
+++ /dev/null
@@ -1,169 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude"
-          xml:id="sec-nix-copy-closure">
-
-<refmeta>
-  <refentrytitle>nix-copy-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-copy-closure</refname>
-  <refpurpose>copy a closure to or from a remote machine via SSH</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-copy-closure</command>
-    <group>
-      <arg choice='plain'><option>--to</option></arg>
-      <arg choice='plain'><option>--from</option></arg>
-    </group>
-    <arg><option>--gzip</option></arg>
-    <!--
-    <arg><option>- -show-progress</option></arg>
-    -->
-    <arg><option>--include-outputs</option></arg>
-    <group>
-      <arg choice='plain'><option>--use-substitutes</option></arg>
-      <arg choice='plain'><option>-s</option></arg>
-    </group>
-    <arg><option>-v</option></arg>
-    <arg choice='plain'>
-      <emphasis>user@</emphasis><emphasis>machine</emphasis>
-    </arg>
-    <arg choice='plain'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para><command>nix-copy-closure</command> gives you an easy and
-efficient way to exchange software between machines.  Given one or
-more Nix store <emphasis>paths</emphasis> on the local
-machine, <command>nix-copy-closure</command> computes the closure of
-those paths (i.e. all their dependencies in the Nix store), and copies
-all paths in the closure to the remote machine via the
-<command>ssh</command> (Secure Shell) command.  With the
-<option>--from</option>, the direction is reversed:
-the closure of <emphasis>paths</emphasis> on a remote machine is
-copied to the Nix store on the local machine.</para>
-
-<para>This command is efficient because it only sends the store paths
-that are missing on the target machine.</para>
-
-<para>Since <command>nix-copy-closure</command> calls
-<command>ssh</command>, you may be asked to type in the appropriate
-password or passphrase.  In fact, you may be asked
-<emphasis>twice</emphasis> because <command>nix-copy-closure</command>
-currently connects twice to the remote machine, first to get the set
-of paths missing on the target machine, and second to send the dump of
-those paths.  If this bothers you, use
-<command>ssh-agent</command>.</para>
-
-
-<refsection><title>Options</title>
-
-<variablelist>
-
-  <varlistentry><term><option>--to</option></term>
-
-    <listitem><para>Copy the closure of
-    <emphasis>paths</emphasis> from the local Nix store to the
-    Nix store on <emphasis>machine</emphasis>.  This is the
-    default.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--from</option></term>
-
-    <listitem><para>Copy the closure of
-    <emphasis>paths</emphasis> from the Nix store on
-    <emphasis>machine</emphasis> to the local Nix
-    store.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--gzip</option></term>
-
-    <listitem><para>Enable compression of the SSH
-    connection.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--include-outputs</option></term>
-
-    <listitem><para>Also copy the outputs of store derivations
-    included in the closure.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--use-substitutes</option> / <option>-s</option></term>
-
-    <listitem><para>Attempt to download missing paths on the target
-    machine using Nix’s substitute mechanism.  Any paths that cannot
-    be substituted on the target are still copied normally from the
-    source.  This is useful, for instance, if the connection between
-    the source and target machine is slow, but the connection between
-    the target machine and <literal>nixos.org</literal> (the default
-    binary cache server) is fast.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>-v</option></term>
-
-    <listitem><para>Show verbose output.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Environment variables</title>
-
-<variablelist>
-
-  <varlistentry><term><literal>NIX_SSHOPTS</literal></term>
-
-    <listitem><para>Additional options to be passed to
-    <command>ssh</command> on the command line.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>Copy Firefox with all its dependencies to a remote machine:
-
-<screen>
-$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)</screen>
-
-</para>
-
-<para>Copy Subversion from a remote machine and then install it into a
-user environment:
-
-<screen>
-$ nix-copy-closure --from alice@itchy.labs \
-    /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
-$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
-</screen>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-daemon.xml b/doc/manual/command-ref/nix-daemon.xml
deleted file mode 100644
index a2161f033..000000000
--- a/doc/manual/command-ref/nix-daemon.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-daemon">
-
-<refmeta>
-  <refentrytitle>nix-daemon</refentrytitle>
-  <manvolnum>8</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-daemon</refname>
-  <refpurpose>Nix multi-user support daemon</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-daemon</command>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The Nix daemon is necessary in multi-user Nix installations.  It
-performs build actions and other operations on the Nix store on behalf
-of unprivileged users.</para>
-
-
-</refsection>
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
deleted file mode 100644
index 6a97c7e37..000000000
--- a/doc/manual/command-ref/nix-env.xml
+++ /dev/null
@@ -1,1503 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-env">
-
-<refmeta>
-  <refentrytitle>nix-env</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-env</refname>
-  <refpurpose>manipulate or query Nix user environments</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-env</command>
-    <xi:include href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg>
-      <group choice='req'>
-        <arg choice='plain'><option>--file</option></arg>
-        <arg choice='plain'><option>-f</option></arg>
-      </group>
-      <emphasis>path</emphasis>
-    </arg>
-    <arg>
-      <group choice='req'>
-        <arg choice='plain'><option>--profile</option></arg>
-        <arg choice='plain'><option>-p</option></arg>
-      </group>
-      <emphasis>path</emphasis>
-    </arg>
-    <arg>
-      <arg choice='plain'><option>--system-filter</option></arg>
-      <emphasis>system</emphasis>
-    </arg>
-    <arg><option>--dry-run</option></arg>
-    <arg choice='plain'><emphasis>operation</emphasis></arg>
-    <arg rep='repeat'><emphasis>options</emphasis></arg>
-    <arg rep='repeat'><emphasis>arguments</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-env</command> is used to manipulate Nix
-user environments.  User environments are sets of software packages
-available to a user at some point in time.  In other words, they are a
-synthesised view of the programs available in the Nix store.  There
-may be many user environments: different users can have different
-environments, and individual users can switch between different
-environments.</para>
-
-<para><command>nix-env</command> takes exactly one
-<emphasis>operation</emphasis> flag which indicates the subcommand to
-be performed.  These are documented below.</para>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Selectors</title>
-
-<para>Several commands, such as <command>nix-env -q</command> and
-<command>nix-env -i</command>, take a list of arguments that specify
-the packages on which to operate. These are extended regular
-expressions that must match the entire name of the package. (For
-details on regular expressions, see
-<citerefentry><refentrytitle>regex</refentrytitle><manvolnum>7</manvolnum></citerefentry>.)
-The match is case-sensitive. The regular expression can optionally be
-followed by a dash and a version number; if omitted, any version of
-the package will match.  Here are some examples:
-
-<variablelist>
-
-  <varlistentry>
-    <term><literal>firefox</literal></term>
-    <listitem><para>Matches the package name
-    <literal>firefox</literal> and any version.</para></listitem>
-  </varlistentry>
-
-  <varlistentry>
-    <term><literal>firefox-32.0</literal></term>
-    <listitem><para>Matches the package name
-    <literal>firefox</literal> and version
-    <literal>32.0</literal>.</para></listitem>
-  </varlistentry>
-
-  <varlistentry>
-    <term><literal>gtk\\+</literal></term>
-    <listitem><para>Matches the package name
-    <literal>gtk+</literal>. The <literal>+</literal> character must
-    be escaped using a backslash to prevent it from being interpreted
-    as a quantifier, and the backslash must be escaped in turn with
-    another backslash to ensure that the shell passes it
-    on.</para></listitem>
-  </varlistentry>
-
-  <varlistentry>
-    <term><literal>.\*</literal></term>
-    <listitem><para>Matches any package name. This is the default for
-    most commands.</para></listitem>
-  </varlistentry>
-
-  <varlistentry>
-    <term><literal>'.*zip.*'</literal></term>
-    <listitem><para>Matches any package name containing the string
-    <literal>zip</literal>. Note the dots: <literal>'*zip*'</literal>
-    does not work, because in a regular expression, the character
-    <literal>*</literal> is interpreted as a
-    quantifier.</para></listitem>
-  </varlistentry>
-
-  <varlistentry>
-    <term><literal>'.*(firefox|chromium).*'</literal></term>
-    <listitem><para>Matches any package name containing the strings
-    <literal>firefox</literal> or
-    <literal>chromium</literal>.</para></listitem>
-  </varlistentry>
-
-</variablelist>
-
-</para>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Common options</title>
-
-<para>This section lists the options that are common to all
-operations.  These options are allowed for every subcommand, though
-they may not always have an effect.  <phrase condition="manual">See
-also <xref linkend="sec-common-options" />.</phrase></para>
-
-<variablelist>
-
-  <varlistentry><term><option>--file</option> / <option>-f</option> <emphasis>path</emphasis></term>
-
-    <listitem><para>Specifies the Nix expression (designated below as
-    the <emphasis>active Nix expression</emphasis>) used by the
-    <option>--install</option>, <option>--upgrade</option>, and
-    <option>--query --available</option> operations to obtain
-    derivations.  The default is
-    <filename>~/.nix-defexpr</filename>.</para>
-
-    <para>If the argument starts with <literal>http://</literal> or
-    <literal>https://</literal>, it is interpreted as the URL of a
-    tarball that will be downloaded and unpacked to a temporary
-    location. The tarball must include a single top-level directory
-    containing at least a file named <filename>default.nix</filename>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--profile</option> / <option>-p</option> <emphasis>path</emphasis></term>
-
-    <listitem><para>Specifies the profile to be used by those
-    operations that operate on a profile (designated below as the
-    <emphasis>active profile</emphasis>).  A profile is a sequence of
-    user environments called <emphasis>generations</emphasis>, one of
-    which is the <emphasis>current
-    generation</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--dry-run</option></term>
-
-    <listitem><para>For the <option>--install</option>,
-    <option>--upgrade</option>, <option>--uninstall</option>,
-    <option>--switch-generation</option>,
-    <option>--delete-generations</option> and
-    <option>--rollback</option> operations, this flag will cause
-    <command>nix-env</command> to print what
-    <emphasis>would</emphasis> be done if this flag had not been
-    specified, without actually doing it.</para>
-
-    <para><option>--dry-run</option> also prints out which paths will
-    be <link linkend="gloss-substitute">substituted</link> (i.e.,
-    downloaded) and which paths will be built from source (because no
-    substitute is available).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--system-filter</option> <emphasis>system</emphasis></term>
-
-    <listitem><para>By default, operations such as <option>--query
-    --available</option> show derivations matching any platform.  This
-    option allows you to use derivations for the specified platform
-    <emphasis>system</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<variablelist condition="manpage">
-  <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Files</title>
-
-<variablelist>
-
-  <varlistentry><term><filename>~/.nix-defexpr</filename></term>
-
-    <listitem><para>The source for the default Nix
-    expressions used by the <option>--install</option>,
-    <option>--upgrade</option>, and <option>--query
-    --available</option> operations to obtain derivations. The
-    <option>--file</option> option may be used to override this
-    default.</para>
-
-    <para>If <filename>~/.nix-defexpr</filename> is a file,
-    it is loaded as a Nix expression. If the expression
-    is a set, it is used as the default Nix expression.
-    If the expression is a function, an empty set is passed
-    as argument and the return value is used as
-    the default Nix expression.</para>
-
-    <para>If <filename>~/.nix-defexpr</filename> is a directory
-    containing a <filename>default.nix</filename> file, that file
-    is loaded as in the above paragraph.</para>
-
-    <para>If <filename>~/.nix-defexpr</filename> is a directory without
-    a <filename>default.nix</filename> file, then its contents
-    (both files and subdirectories) are loaded as Nix expressions.
-    The expressions are combined into a single set, each expression
-    under an attribute with the same name as the original file
-    or subdirectory.
-    </para>
-
-    <para>For example, if <filename>~/.nix-defexpr</filename> contains
-    two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>,
-    then the default Nix expression will essentially be
-
-<programlisting>
-{
-  foo = import ~/.nix-defexpr/foo.nix;
-  bar = import ~/.nix-defexpr/bar.nix;
-}</programlisting>
-
-    </para>
-
-    <para>The file <filename>manifest.nix</filename> is always ignored.
-    Subdirectories without a <filename>default.nix</filename> file
-    are traversed recursively in search of more Nix expressions,
-    but the names of these intermediate directories are not
-    added to the attribute paths of the default Nix expression.</para>
-
-    <para>The command <command>nix-channel</command> places symlinks
-    to the downloaded Nix expressions from each subscribed channel in
-    this directory.</para>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><filename>~/.nix-profile</filename></term>
-
-    <listitem><para>A symbolic link to the user's current profile.  By
-    default, this symlink points to
-    <filename><emphasis>prefix</emphasis>/var/nix/profiles/default</filename>.
-    The <literal>PATH</literal> environment variable should include
-    <filename>~/.nix-profile/bin</filename> for the user environment
-    to be visible to the user.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id="rsec-nix-env-install"><title>Operation <option>--install</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--install</option></arg>
-    <arg choice='plain'><option>-i</option></arg>
-  </group>
-  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-inst-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-  <group choice='opt'>
-    <arg choice='plain'><option>--preserve-installed</option></arg>
-    <arg choice='plain'><option>-P</option></arg>
-  </group>
-  <group choice='opt'>
-    <arg choice='plain'><option>--remove-all</option></arg>
-    <arg choice='plain'><option>-r</option></arg>
-  </group>
-  <arg choice='plain' rep='repeat'><emphasis>args</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>The install operation creates a new user environment, based on
-the current generation of the active profile, to which a set of store
-paths described by <emphasis>args</emphasis> is added.  The
-arguments <emphasis>args</emphasis> map to store paths in a
-number of possible ways:
-
-<itemizedlist>
-
-  <listitem><para>By default, <emphasis>args</emphasis> is a set
-  of derivation names denoting derivations in the active Nix
-  expression.  These are realised, and the resulting output paths are
-  installed.  Currently installed derivations with a name equal to the
-  name of a derivation being added are removed unless the option
-  <option>--preserve-installed</option> is
-  specified.</para>
-
-  <para>If there are multiple derivations matching a name in
-  <emphasis>args</emphasis> that have the same name (e.g.,
-  <literal>gcc-3.3.6</literal> and <literal>gcc-4.1.1</literal>), then
-  the derivation with the highest <emphasis>priority</emphasis> is
-  used.  A derivation can define a priority by declaring the
-  <varname>meta.priority</varname> attribute.  This attribute should
-  be a number, with a higher value denoting a lower priority.  The
-  default priority is <literal>0</literal>.</para>
-
-  <para>If there are multiple matching derivations with the same
-  priority, then the derivation with the highest version will be
-  installed.</para>
-
-  <para>You can force the installation of multiple derivations with
-  the same name by being specific about the versions.  For instance,
-  <literal>nix-env -i gcc-3.3.6 gcc-4.1.1</literal> will install both
-  version of GCC (and will probably cause a user environment
-  conflict!).</para></listitem>
-
-  <listitem><para>If <link
-  linkend='opt-attr'><option>--attr</option></link>
-  (<option>-A</option>) is specified, the arguments are
-  <emphasis>attribute paths</emphasis> that select attributes from the
-  top-level Nix expression.  This is faster than using derivation
-  names and unambiguous.  To find out the attribute paths of available
-  packages, use <literal>nix-env -qaP</literal>.</para></listitem>
-
-  <listitem><para>If <option>--from-profile</option>
-  <emphasis>path</emphasis> is given,
-  <emphasis>args</emphasis> is a set of names denoting installed
-  store paths in the profile <emphasis>path</emphasis>.  This is
-  an easy way to copy user environment elements from one profile to
-  another.</para></listitem>
-
-  <listitem><para>If <option>--from-expression</option> is given,
-  <emphasis>args</emphasis> are Nix <link
-  linkend="ss-functions">functions</link> that are called with the
-  active Nix expression as their single argument.  The derivations
-  returned by those function calls are installed.  This allows
-  derivations to be specified in an unambiguous way, which is necessary
-  if there are multiple derivations with the same
-  name.</para></listitem>
-
-  <listitem><para>If <emphasis>args</emphasis> are store
-  derivations, then these are <link
-  linkend="rsec-nix-store-realise">realised</link>, and the resulting
-  output paths are installed.</para></listitem>
-
-  <listitem><para>If <emphasis>args</emphasis> are store paths
-  that are not store derivations, then these are <link
-  linkend="rsec-nix-store-realise">realised</link> and
-  installed.</para></listitem>
-
-  <listitem><para>By default all outputs are installed for each derivation.
-  That can be reduced by setting <literal>meta.outputsToInstall</literal>.
-  </para></listitem> <!-- TODO: link nixpkgs docs on the ability to override those. -->
-
-</itemizedlist>
-
-</para>
-
-</refsection>
-
-
-<refsection><title>Flags</title>
-
-<variablelist>
-
-  <varlistentry><term><option>--prebuilt-only</option> / <option>-b</option></term>
-
-    <listitem><para>Use only derivations for which a substitute is
-    registered, i.e., there is a pre-built binary available that can
-    be downloaded in lieu of building the derivation.  Thus, no
-    packages will be built from source.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--preserve-installed</option></term>
-    <term><option>-P</option></term>
-
-    <listitem><para>Do not remove derivations with a name matching one
-    of the derivations being installed.  Usually, trying to have two
-    versions of the same package installed in the same generation of a
-    profile will lead to an error in building the generation, due to
-    file name clashes between the two versions.  However, this is not
-    the case for all packages.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--remove-all</option></term>
-    <term><option>-r</option></term>
-
-    <listitem><para>Remove all previously installed packages first.
-    This is equivalent to running <literal>nix-env -e '.*'</literal>
-    first, except that everything happens in a single
-    transaction.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection xml:id='refsec-nix-env-install-examples'><title>Examples</title>
-
-<para>To install a specific version of <command>gcc</command> from the
-active Nix expression:
-
-<screen>
-$ nix-env --install gcc-3.3.2
-installing `gcc-3.3.2'
-uninstalling `gcc-3.1'</screen>
-
-Note the previously installed version is removed, since
-<option>--preserve-installed</option> was not specified.</para>
-
-<para>To install an arbitrary version:
-
-<screen>
-$ nix-env --install gcc
-installing `gcc-3.3.2'</screen>
-
-</para>
-
-<para>To install using a specific attribute:
-
-<screen>
-$ nix-env -i -A gcc40mips
-$ nix-env -i -A xorg.xorgserver</screen>
-
-</para>
-
-<para>To install all derivations in the Nix expression <filename>foo.nix</filename>:
-
-<screen>
-$ nix-env -f ~/foo.nix -i '.*'</screen>
-
-</para>
-
-<para>To copy the store path with symbolic name <literal>gcc</literal>
-from another profile:
-
-<screen>
-$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc</screen>
-
-</para>
-
-<para>To install a specific store derivation (typically created by
-<command>nix-instantiate</command>):
-
-<screen>
-$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv</screen>
-
-</para>
-
-<para>To install a specific output path:
-
-<screen>
-$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3</screen>
-
-</para>
-
-<para>To install from a Nix expression specified on the command-line:
-
-<screen>
-$ nix-env -f ./foo.nix -i -E \
-    'f: (f {system = "i686-linux";}).subversionWithJava'</screen>
-
-I.e., this evaluates to <literal>(f: (f {system =
-"i686-linux";}).subversionWithJava) (import ./foo.nix)</literal>, thus
-selecting the <literal>subversionWithJava</literal> attribute from the
-set returned by calling the function defined in
-<filename>./foo.nix</filename>.</para>
-
-<para>A dry-run tells you which paths will be downloaded or built from
-source:
-
-<screen>
-$ nix-env -f '&lt;nixpkgs>' -iA hello --dry-run
-(dry run; not doing anything)
-installing ‘hello-2.10’
-this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
-  /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
-  <emphasis>...</emphasis></screen>
-
-</para>
-
-<para>To install Firefox from the latest revision in the Nixpkgs/NixOS
-14.12 channel:
-
-<screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
-</screen>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id="rsec-nix-env-upgrade"><title>Operation <option>--upgrade</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--upgrade</option></arg>
-    <arg choice='plain'><option>-u</option></arg>
-  </group>
-  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-inst-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-  <group choice='opt'>
-    <arg choice='plain'><option>--lt</option></arg>
-    <arg choice='plain'><option>--leq</option></arg>
-    <arg choice='plain'><option>--eq</option></arg>
-    <arg choice='plain'><option>--always</option></arg>
-  </group>
-  <arg choice='plain' rep='repeat'><emphasis>args</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The upgrade operation creates a new user environment, based on
-the current generation of the active profile, in which all store paths
-are replaced for which there are newer versions in the set of paths
-described by <emphasis>args</emphasis>.  Paths for which there
-are no newer versions are left untouched; this is not an error.  It is
-also not an error if an element of <emphasis>args</emphasis>
-matches no installed derivations.</para>
-
-<para>For a description of how <emphasis>args</emphasis> is
-mapped to a set of store paths, see <link
-linkend="rsec-nix-env-install"><option>--install</option></link>.  If
-<emphasis>args</emphasis> describes multiple store paths with
-the same symbolic name, only the one with the highest version is
-installed.</para>
-
-</refsection>
-
-<refsection><title>Flags</title>
-
-<variablelist>
-
-  <varlistentry><term><option>--lt</option></term>
-
-    <listitem><para>Only upgrade a derivation to newer versions.  This
-    is the default.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--leq</option></term>
-
-    <listitem><para>In addition to upgrading to newer versions, also
-    “upgrade” to derivations that have the same version.  Version are
-    not a unique identification of a derivation, so there may be many
-    derivations that have the same version.  This flag may be useful
-    to force “synchronisation” between the installed and available
-    derivations.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--eq</option></term>
-
-    <listitem><para><emphasis>Only</emphasis> “upgrade” to derivations
-    that have the same version.  This may not seem very useful, but it
-    actually is, e.g., when there is a new release of Nixpkgs and you
-    want to replace installed applications with the same versions
-    built against newer dependencies (to reduce the number of
-    dependencies floating around on your system).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--always</option></term>
-
-    <listitem><para>In addition to upgrading to newer versions, also
-    “upgrade” to derivations that have the same or a lower version.
-    I.e., derivations may actually be downgraded depending on what is
-    available in the active Nix expression.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<para>For the other flags, see <option
-linkend="rsec-nix-env-install">--install</option>.</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env --upgrade gcc
-upgrading `gcc-3.3.1' to `gcc-3.4'
-
-$ nix-env -u gcc-3.3.2 --always <lineannotation>(switch to a specific version)</lineannotation>
-upgrading `gcc-3.4' to `gcc-3.3.2'
-
-$ nix-env --upgrade pan
-<lineannotation>(no upgrades available, so nothing happens)</lineannotation>
-
-$ nix-env -u <lineannotation>(try to upgrade everything)</lineannotation>
-upgrading `hello-2.1.2' to `hello-2.1.3'
-upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
-
-</refsection>
-
-<refsection xml:id="ssec-version-comparisons"><title>Versions</title>
-
-<para>The upgrade operation determines whether a derivation
-<varname>y</varname> is an upgrade of a derivation
-<varname>x</varname> by looking at their respective
-<literal>name</literal> attributes.  The names (e.g.,
-<literal>gcc-3.3.1</literal> are split into two parts: the package
-name (<literal>gcc</literal>), and the version
-(<literal>3.3.1</literal>).  The version part starts after the first
-dash not followed by a letter.  <varname>x</varname> is considered an
-upgrade of <varname>y</varname> if their package names match, and the
-version of <varname>y</varname> is higher that that of
-<varname>x</varname>.</para>
-
-<para>The versions are compared by splitting them into contiguous
-components of numbers and letters.  E.g., <literal>3.3.1pre5</literal>
-is split into <literal>[3, 3, 1, "pre", 5]</literal>.  These lists are
-then compared lexicographically (from left to right).  Corresponding
-components <varname>a</varname> and <varname>b</varname> are compared
-as follows.  If they are both numbers, integer comparison is used.  If
-<varname>a</varname> is an empty string and <varname>b</varname> is a
-number, <varname>a</varname> is considered less than
-<varname>b</varname>.  The special string component
-<literal>pre</literal> (for <emphasis>pre-release</emphasis>) is
-considered to be less than other components.  String components are
-considered less than number components.  Otherwise, they are compared
-lexicographically (i.e., using case-sensitive string comparison).</para>
-
-<para>This is illustrated by the following examples:
-
-<screen>
-1.0 &lt; 2.3
-2.1 &lt; 2.3
-2.3 = 2.3
-2.5 > 2.3
-3.1 > 2.3
-2.3.1 > 2.3
-2.3.1 > 2.3a
-2.3pre1 &lt; 2.3
-2.3pre3 &lt; 2.3pre12
-2.3a &lt; 2.3c
-2.3pre1 &lt; 2.3c
-2.3pre1 &lt; 2.3q</screen>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--uninstall</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--uninstall</option></arg>
-    <arg choice='plain'><option>-e</option></arg>
-  </group>
-  <arg choice='plain' rep='repeat'><emphasis>drvnames</emphasis></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The uninstall operation creates a new user environment, based on
-the current generation of the active profile, from which the store
-paths designated by the symbolic names
-<emphasis>names</emphasis> are removed.</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env --uninstall gcc
-$ nix-env -e '.*' <lineannotation>(remove everything)</lineannotation></screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id="rsec-nix-env-set"><title>Operation <option>--set</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <arg choice='plain'><option>--set</option></arg>
-  <arg choice='plain'><emphasis>drvname</emphasis></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The <option>--set</option> operation modifies the current generation of a
-profile so that it contains exactly the specified derivation, and nothing else.
-</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<para>
-The following updates a profile such that its current generation will contain
-just Firefox:
-
-<screen>
-$ nix-env -p /nix/var/nix/profiles/browser --set firefox</screen>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id="rsec-nix-env-set-flag"><title>Operation <option>--set-flag</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <arg choice='plain'><option>--set-flag</option></arg>
-  <arg choice='plain'><emphasis>name</emphasis></arg>
-  <arg choice='plain'><emphasis>value</emphasis></arg>
-  <arg choice='plain' rep='repeat'><emphasis>drvnames</emphasis></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The <option>--set-flag</option> operation allows meta attributes
-of installed packages to be modified.  There are several attributes
-that can be usefully modified, because they affect the behaviour of
-<command>nix-env</command> or the user environment build
-script:
-
-<itemizedlist>
-
-  <listitem><para><varname>priority</varname> can be changed to
-  resolve filename clashes.  The user environment build script uses
-  the <varname>meta.priority</varname> attribute of derivations to
-  resolve filename collisions between packages.  Lower priority values
-  denote a higher priority.  For instance, the GCC wrapper package and
-  the Binutils package in Nixpkgs both have a file
-  <filename>bin/ld</filename>, so previously if you tried to install
-  both you would get a collision.  Now, on the other hand, the GCC
-  wrapper declares a higher priority than Binutils, so the former’s
-  <filename>bin/ld</filename> is symlinked in the user
-  environment.</para></listitem>
-
-  <listitem><para><varname>keep</varname> can be set to
-  <literal>true</literal> to prevent the package from being upgraded
-  or replaced.  This is useful if you want to hang on to an older
-  version of a package.</para></listitem>
-
-  <listitem><para><varname>active</varname> can be set to
-  <literal>false</literal> to “disable” the package.  That is, no
-  symlinks will be generated to the files of the package, but it
-  remains part of the profile (so it won’t be garbage-collected).  It
-  can be set back to <literal>true</literal> to re-enable the
-  package.</para></listitem>
-
-</itemizedlist>
-
-</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<para>To prevent the currently installed Firefox from being upgraded:
-
-<screen>
-$ nix-env --set-flag keep true firefox</screen>
-
-After this, <command>nix-env -u</command> will ignore Firefox.</para>
-
-<para>To disable the currently installed Firefox, then install a new
-Firefox while the old remains part of the profile:
-
-<screen>
-$ nix-env -q
-firefox-2.0.0.9 <lineannotation>(the current one)</lineannotation>
-
-$ nix-env --preserve-installed -i firefox-2.0.0.11
-installing `firefox-2.0.0.11'
-building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
-collision between `/nix/store/<emphasis>...</emphasis>-firefox-2.0.0.11/bin/firefox'
-  and `/nix/store/<emphasis>...</emphasis>-firefox-2.0.0.9/bin/firefox'.
-<lineannotation>(i.e., can’t have two active at the same time)</lineannotation>
-
-$ nix-env --set-flag active false firefox
-setting flag on `firefox-2.0.0.9'
-
-$ nix-env --preserve-installed -i firefox-2.0.0.11
-installing `firefox-2.0.0.11'
-
-$ nix-env -q
-firefox-2.0.0.11 <lineannotation>(the enabled one)</lineannotation>
-firefox-2.0.0.9 <lineannotation>(the disabled one)</lineannotation></screen>
-
-</para>
-
-<para>To make files from <literal>binutils</literal> take precedence
-over files from <literal>gcc</literal>:
-
-<screen>
-$ nix-env --set-flag priority 5 binutils
-$ nix-env --set-flag priority 10 gcc</screen>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--query</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--query</option></arg>
-    <arg choice='plain'><option>-q</option></arg>
-  </group>
-  <group choice='opt'>
-    <arg choice='plain'><option>--installed</option></arg>
-    <arg choice='plain'><option>--available</option></arg>
-    <arg choice='plain'><option>-a</option></arg>
-  </group>
-
-  <sbr />
-
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--status</option></arg>
-      <arg choice='plain'><option>-s</option></arg>
-    </group>
-  </arg>
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--attr-path</option></arg>
-      <arg choice='plain'><option>-P</option></arg>
-    </group>
-  </arg>
-  <arg><option>--no-name</option></arg>
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--compare-versions</option></arg>
-      <arg choice='plain'><option>-c</option></arg>
-    </group>
-  </arg>
-  <arg><option>--system</option></arg>
-  <arg><option>--drv-path</option></arg>
-  <arg><option>--out-path</option></arg>
-  <arg><option>--description</option></arg>
-  <arg><option>--meta</option></arg>
-
-  <sbr />
-
-  <arg><option>--xml</option></arg>
-  <arg><option>--json</option></arg>
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--prebuilt-only</option></arg>
-      <arg choice='plain'><option>-b</option></arg>
-    </group>
-  </arg>
-
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--attr</option></arg>
-      <arg choice='plain'><option>-A</option></arg>
-    </group>
-    <emphasis>attribute-path</emphasis>
-  </arg>
-
-  <sbr />
-
-  <arg choice='plain' rep='repeat'><emphasis>names</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>The query operation displays information about either the store
-paths that are installed in the current generation of the active
-profile (<option>--installed</option>), or the derivations that are
-available for installation in the active Nix expression
-(<option>--available</option>).  It only prints information about
-derivations whose symbolic name matches one of
-<emphasis>names</emphasis>.</para>
-
-<para>The derivations are sorted by their <literal>name</literal>
-attributes.</para>
-
-</refsection>
-
-
-<refsection><title>Source selection</title>
-
-<para>The following flags specify the set of things on which the query
-operates.</para>
-
-<variablelist>
-
-  <varlistentry><term><option>--installed</option></term>
-
-    <listitem><para>The query operates on the store paths that are
-    installed in the current generation of the active profile.  This
-    is the default.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--available</option></term>
-    <term><option>-a</option></term>
-
-    <listitem><para>The query operates on the derivations that are
-    available in the active Nix expression.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Queries</title>
-
-<para>The following flags specify what information to display about
-the selected derivations.  Multiple flags may be specified, in which
-case the information is shown in the order given here.  Note that the
-name of the derivation is shown unless <option>--no-name</option> is
-specified.</para>
-
-<!-- TODO: fix the terminology here; i.e., derivations, store paths,
-user environment elements, etc. -->
-
-<variablelist>
-
-  <varlistentry><term><option>--xml</option></term>
-
-    <listitem><para>Print the result in an XML representation suitable
-    for automatic processing by other tools.  The root element is
-    called <literal>items</literal>, which contains a
-    <literal>item</literal> element for each available or installed
-    derivation.  The fields discussed below are all stored in
-    attributes of the <literal>item</literal>
-    elements.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--json</option></term>
-
-    <listitem><para>Print the result in a JSON representation suitable
-    for automatic processing by other tools.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--prebuilt-only</option> / <option>-b</option></term>
-
-    <listitem><para>Show only derivations for which a substitute is
-    registered, i.e., there is a pre-built binary available that can
-    be downloaded in lieu of building the derivation.  Thus, this
-    shows all packages that probably can be installed
-    quickly.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--status</option></term>
-    <term><option>-s</option></term>
-
-    <listitem><para>Print the <emphasis>status</emphasis> of the
-    derivation.  The status consists of three characters.  The first
-    is <literal>I</literal> or <literal>-</literal>, indicating
-    whether the derivation is currently installed in the current
-    generation of the active profile.  This is by definition the case
-    for <option>--installed</option>, but not for
-    <option>--available</option>.  The second is <literal>P</literal>
-    or <literal>-</literal>, indicating whether the derivation is
-    present on the system.  This indicates whether installation of an
-    available derivation will require the derivation to be built.  The
-    third is <literal>S</literal> or <literal>-</literal>, indicating
-    whether a substitute is available for the
-    derivation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--attr-path</option></term>
-    <term><option>-P</option></term>
-
-    <listitem><para>Print the <emphasis>attribute path</emphasis> of
-    the derivation, which can be used to unambiguously select it using
-    the <link linkend="opt-attr"><option>--attr</option> option</link>
-    available in commands that install derivations like
-    <literal>nix-env --install</literal>. This option only works
-    together with <option>--available</option></para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--no-name</option></term>
-
-    <listitem><para>Suppress printing of the <literal>name</literal>
-    attribute of each derivation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--compare-versions</option> /
-  <option>-c</option></term>
-
-    <listitem><para>Compare installed versions to available versions,
-    or vice versa (if <option>--available</option> is given).  This is
-    useful for quickly seeing whether upgrades for installed
-    packages are available in a Nix expression.  A column is added
-    with the following meaning:
-
-    <variablelist>
-
-      <varlistentry><term><literal>&lt;</literal> <emphasis>version</emphasis></term>
-
-        <listitem><para>A newer version of the package is available
-        or installed.</para></listitem>
-
-      </varlistentry>
-
-      <varlistentry><term><literal>=</literal> <emphasis>version</emphasis></term>
-
-        <listitem><para>At most the same version of the package is
-        available or installed.</para></listitem>
-
-      </varlistentry>
-
-      <varlistentry><term><literal>></literal> <emphasis>version</emphasis></term>
-
-        <listitem><para>Only older versions of the package are
-        available or installed.</para></listitem>
-
-      </varlistentry>
-
-      <varlistentry><term><literal>- ?</literal></term>
-
-        <listitem><para>No version of the package is available or
-        installed.</para></listitem>
-
-      </varlistentry>
-
-    </variablelist>
-
-    </para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--system</option></term>
-
-    <listitem><para>Print the <literal>system</literal> attribute of
-    the derivation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--drv-path</option></term>
-
-    <listitem><para>Print the path of the store
-    derivation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--out-path</option></term>
-
-    <listitem><para>Print the output path of the
-    derivation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--description</option></term>
-
-    <listitem><para>Print a short (one-line) description of the
-    derivation, if available.  The description is taken from the
-    <literal>meta.description</literal> attribute of the
-    derivation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--meta</option></term>
-
-    <listitem><para>Print all of the meta-attributes of the
-    derivation.  This option is only available with
-    <option>--xml</option> or <option>--json</option>.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To show installed packages:
-
-<screen>
-$ nix-env -q
-bison-1.875c
-docbook-xml-4.2
-firefox-1.0.4
-MPlayer-1.0pre7
-ORBit2-2.8.3
-<emphasis>…</emphasis>
-</screen>
-
-</para>
-
-<para>To show available packages:
-
-<screen>
-$ nix-env -qa
-firefox-1.0.7
-GConf-2.4.0.1
-MPlayer-1.0pre7
-ORBit2-2.8.3
-<emphasis>…</emphasis>
-</screen>
-
-</para>
-
-<para>To show the status of available packages:
-
-<screen>
-$ nix-env -qas
--P- firefox-1.0.7   <lineannotation>(not installed but present)</lineannotation>
---S GConf-2.4.0.1   <lineannotation>(not present, but there is a substitute for fast installation)</lineannotation>
---S MPlayer-1.0pre3 <lineannotation>(i.e., this is not the installed MPlayer, even though the version is the same!)</lineannotation>
-IP- ORBit2-2.8.3    <lineannotation>(installed and by definition present)</lineannotation>
-<emphasis>…</emphasis>
-</screen>
-
-</para>
-
-<para>To show available packages in the Nix expression <filename>foo.nix</filename>:
-
-<screen>
-$ nix-env -f ./foo.nix -qa
-foo-1.2.3
-</screen>
-
-</para>
-
-<para>To compare installed versions to what’s available:
-
-<screen>
-$ nix-env -qc
-<emphasis>...</emphasis>
-acrobat-reader-7.0 - ?      <lineannotation>(package is not available at all)</lineannotation>
-autoconf-2.59      = 2.59   <lineannotation>(same version)</lineannotation>
-firefox-1.0.4      &lt; 1.0.7  <lineannotation>(a more recent version is available)</lineannotation>
-<emphasis>...</emphasis>
-</screen>
-
-</para>
-
-<para>To show all packages with “<literal>zip</literal>” in the name:
-
-<screen>
-$ nix-env -qa '.*zip.*'
-bzip2-1.0.6
-gzip-1.6
-zip-3.0
-<emphasis>…</emphasis>
-</screen>
-
-</para>
-
-<para>To show all packages with “<literal>firefox</literal>” or
-“<literal>chromium</literal>” in the name:
-
-<screen>
-$ nix-env -qa '.*(firefox|chromium).*'
-chromium-37.0.2062.94
-chromium-beta-38.0.2125.24
-firefox-32.0.3
-firefox-with-plugins-13.0.1
-<emphasis>…</emphasis>
-</screen>
-
-</para>
-
-<para>To show all packages in the latest revision of the Nixpkgs
-repository:
-
-<screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
-</screen>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--switch-profile</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--switch-profile</option></arg>
-    <arg choice='plain'><option>-S</option></arg>
-  </group>
-  <arg choice='req'><emphasis>path</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>This operation makes <emphasis>path</emphasis> the current
-profile for the user.  That is, the symlink
-<filename>~/.nix-profile</filename> is made to point to
-<emphasis>path</emphasis>.</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env -S ~/my-profile</screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--list-generations</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <arg choice='plain'><option>--list-generations</option></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>This operation print a list of all the currently existing
-generations for the active profile.  These may be switched to using
-the <option>--switch-generation</option> operation.  It also prints
-the creation date of the generation, and indicates the current
-generation.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env --list-generations
-  95   2004-02-06 11:48:24
-  96   2004-02-06 11:49:01
-  97   2004-02-06 16:22:45
-  98   2004-02-06 16:24:33   (current)</screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--delete-generations</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <arg choice='plain'><option>--delete-generations</option></arg>
-  <arg choice='plain' rep='repeat'><emphasis>generations</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>This operation deletes the specified generations of the current
-profile.  The generations can be a list of generation numbers, the
-special value <literal>old</literal> to delete all non-current
-generations,  a value such as <literal>30d</literal> to delete all
-generations older than the specified number of days (except for the
-generation that was active at that point in time), or a value such as
-<literal>+5</literal> to keep the last <literal>5</literal> generations
-ignoring any newer than current, e.g., if <literal>30</literal> is the current
-generation <literal>+5</literal> will delete generation <literal>25</literal>
-and all older generations.
-Periodically deleting old generations is important to make garbage collection
-effective.</para>
-
-</refsection>
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env --delete-generations 3 4 8
-
-$ nix-env --delete-generations +5
-
-$ nix-env --delete-generations 30d
-
-$ nix-env -p other_profile --delete-generations old</screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--switch-generation</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--switch-generation</option></arg>
-    <arg choice='plain'><option>-G</option></arg>
-  </group>
-  <arg choice='req'><emphasis>generation</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>This operation makes generation number
-<emphasis>generation</emphasis> the current generation of the
-active profile.  That is, if the
-<filename><emphasis>profile</emphasis></filename> is the path to
-the active profile, then the symlink
-<filename><emphasis>profile</emphasis></filename> is made to
-point to
-<filename><emphasis>profile</emphasis>-<emphasis>generation</emphasis>-link</filename>,
-which is in turn a symlink to the actual user environment in the Nix
-store.</para>
-
-<para>Switching will fail if the specified generation does not exist.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env -G 42
-switching from generation 50 to 42</screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--rollback</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <arg choice='plain'><option>--rollback</option></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>This operation switches to the “previous” generation of the
-active profile, that is, the highest numbered generation lower than
-the current generation, if it exists.  It is just a convenience
-wrapper around <option>--list-generations</option> and
-<option>--switch-generation</option>.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env --rollback
-switching from generation 92 to 91
-
-$ nix-env --rollback
-error: no generation older than the current (91) exists</screen>
-
-</refsection>
-
-</refsection>
-
-
-<refsection condition="manpage"><title>Environment variables</title>
-
-<variablelist>
-
-  <varlistentry><term><literal>NIX_PROFILE</literal></term>
-
-    <listitem><para>Location of the Nix profile.  Defaults to the
-    target of the symlink <filename>~/.nix-profile</filename>, if it
-    exists, or <filename>/nix/var/nix/profiles/default</filename>
-    otherwise.</para></listitem>
-
-  </varlistentry>
-
-  <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-hash.xml b/doc/manual/command-ref/nix-hash.xml
deleted file mode 100644
index 2f80dc568..000000000
--- a/doc/manual/command-ref/nix-hash.xml
+++ /dev/null
@@ -1,176 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-hash">
-  
-<refmeta>
-  <refentrytitle>nix-hash</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-hash</refname>
-  <refpurpose>compute the cryptographic hash of a path</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-hash</command>
-    <arg><option>--flat</option></arg>
-    <arg><option>--base32</option></arg>
-    <arg><option>--truncate</option></arg>
-    <arg><option>--type</option> <emphasis>hashAlgo</emphasis></arg>
-    <arg choice='plain' rep='repeat'><emphasis>path</emphasis></arg>
-  </cmdsynopsis>
-  <cmdsynopsis>
-    <command>nix-hash</command>
-    <arg choice='plain'><option>--to-base16</option></arg>
-    <arg choice='plain' rep='repeat'><emphasis>hash</emphasis></arg>
-  </cmdsynopsis>
-  <cmdsynopsis>
-    <command>nix-hash</command>
-    <arg choice='plain'><option>--to-base32</option></arg>
-    <arg choice='plain' rep='repeat'><emphasis>hash</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-hash</command> computes the
-cryptographic hash of the contents of each
-<emphasis>path</emphasis> and prints it on standard output.  By
-default, it computes an MD5 hash, but other hash algorithms are
-available as well.  The hash is printed in hexadecimal.  To generate
-the same hash as <command>nix-prefetch-url</command> you have to
-specify multiple arguments, see below for an example.</para>
-
-<para>The hash is computed over a <emphasis>serialisation</emphasis>
-of each path: a dump of the file system tree rooted at the path.  This
-allows directories and symlinks to be hashed as well as regular files.
-The dump is in the <emphasis>NAR format</emphasis> produced by <link
-linkend="refsec-nix-store-dump"><command>nix-store</command>
-<option>--dump</option></link>.  Thus, <literal>nix-hash
-<emphasis>path</emphasis></literal> yields the same
-cryptographic hash as <literal>nix-store --dump
-<emphasis>path</emphasis> | md5sum</literal>.</para>
-
-</refsection>
-
-
-<refsection><title>Options</title>
-
-<variablelist>
-  
-  <varlistentry><term><option>--flat</option></term>
-
-    <listitem><para>Print the cryptographic hash of the contents of
-    each regular file <emphasis>path</emphasis>.  That is, do
-    not compute the hash over the dump of
-    <emphasis>path</emphasis>.  The result is identical to that
-    produced by the GNU commands <command>md5sum</command> and
-    <command>sha1sum</command>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--base32</option></term>
-
-    <listitem><para>Print the hash in a base-32 representation rather
-    than hexadecimal.  This base-32 representation is more compact and
-    can be used in Nix expressions (such as in calls to
-    <function>fetchurl</function>).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--truncate</option></term>
-
-    <listitem><para>Truncate hashes longer than 160 bits (such as
-    SHA-256) to 160 bits.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--type</option> <emphasis>hashAlgo</emphasis></term>
-
-    <listitem><para>Use the specified cryptographic hash algorithm,
-    which can be one of <literal>md5</literal>,
-    <literal>sha1</literal>, and
-    <literal>sha256</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--to-base16</option></term>
-
-    <listitem><para>Don’t hash anything, but convert the base-32 hash
-    representation <emphasis>hash</emphasis> to
-    hexadecimal.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--to-base32</option></term>
-
-    <listitem><para>Don’t hash anything, but convert the hexadecimal
-    hash representation <emphasis>hash</emphasis> to
-    base-32.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>Computing the same hash as <command>nix-prefetch-url</command>:
-<screen>
-$ nix-prefetch-url file://&lt;(echo test)
-1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
-$ nix-hash --type sha256 --flat --base32 &lt;(echo test)
-1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
-</screen>
-</para>
-
-<para>Computing hashes:
-
-<screen>
-$ mkdir test
-$ echo "hello" > test/world
-
-$ nix-hash test/ <lineannotation>(MD5 hash; default)</lineannotation>
-8179d3caeff1869b5ba1744e5a245c04
-
-$ nix-store --dump test/ | md5sum <lineannotation>(for comparison)</lineannotation>
-8179d3caeff1869b5ba1744e5a245c04  -
-
-$ nix-hash --type sha1 test/
-e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
-
-$ nix-hash --type sha1 --base32 test/
-nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
-
-$ nix-hash --type sha256 --flat test/
-error: reading file `test/': Is a directory
-
-$ nix-hash --type sha256 --flat test/world
-5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03</screen>
-
-</para>
-
-<para>Converting between hexadecimal and base-32:
-
-<screen>
-$ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
-nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
-
-$ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
-e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6</screen>
-
-</para>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
deleted file mode 100644
index dc055fa51..000000000
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ /dev/null
@@ -1,266 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-instantiate">
-
-<refmeta>
-  <refentrytitle>nix-instantiate</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-instantiate</refname>
-  <refpurpose>instantiate store derivations from Nix expressions</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-instantiate</command>
-    <group>
-      <arg choice='plain'><option>--parse</option></arg>
-      <arg choice='plain'>
-        <option>--eval</option>
-        <arg><option>--strict</option></arg>
-        <arg><option>--json</option></arg>
-        <arg><option>--xml</option></arg>
-      </arg>
-    </group>
-    <arg><option>--read-write-mode</option></arg>
-    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg>
-      <group choice='req'>
-        <arg choice='plain'><option>--attr</option></arg>
-        <arg choice='plain'><option>-A</option></arg>
-      </group>
-      <emphasis>attrPath</emphasis>
-    </arg>
-    <arg><option>--add-root</option> <emphasis>path</emphasis></arg>
-    <arg><option>--indirect</option></arg>
-    <group>
-      <arg choice='plain'><option>--expr</option></arg>
-      <arg choice='plain'><option>-E</option></arg>
-    </group>
-    <arg choice='plain' rep='repeat'><emphasis>files</emphasis></arg>
-  </cmdsynopsis>
-  <cmdsynopsis>
-    <command>nix-instantiate</command>
-    <arg choice='plain'><option>--find-file</option></arg>
-    <arg choice='plain' rep='repeat'><emphasis>files</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-instantiate</command> generates <link
-linkend="gloss-derivation">store derivations</link> from (high-level)
-Nix expressions.  It evaluates the Nix expressions in each of
-<emphasis>files</emphasis> (which defaults to
-<emphasis>./default.nix</emphasis>).  Each top-level expression
-should evaluate to a derivation, a list of derivations, or a set of
-derivations.  The paths of the resulting store derivations are printed
-on standard output.</para>
-
-<para>If <emphasis>files</emphasis> is the character
-<literal>-</literal>, then a Nix expression will be read from standard
-input.</para>
-
-<para condition="manual">See also <xref linkend="sec-common-options"
-/> for a list of common options.</para>
-
-</refsection>
-
-
-<refsection><title>Options</title>
-
-<variablelist>
-
-  <varlistentry>
-    <term><option>--add-root</option> <emphasis>path</emphasis></term>
-    <term><option>--indirect</option></term>
-
-    <listitem><para>See the <link linkend="opt-add-root">corresponding
-    options</link> in <command>nix-store</command>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--parse</option></term>
-
-    <listitem><para>Just parse the input files, and print their
-    abstract syntax trees on standard output in ATerm
-    format.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--eval</option></term>
-
-    <listitem><para>Just parse and evaluate the input files, and print
-    the resulting values on standard output.  No instantiation of
-    store derivations takes place.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--find-file</option></term>
-
-    <listitem><para>Look up the given files in Nix’s search path (as
-    specified by the <literal linkend="env-NIX_PATH">NIX_PATH</literal>
-    environment variable).  If found, print the corresponding absolute
-    paths on standard output.  For instance, if
-    <literal>NIX_PATH</literal> is
-    <literal>nixpkgs=/home/alice/nixpkgs</literal>, then
-    <literal>nix-instantiate --find-file nixpkgs/default.nix</literal>
-    will print
-    <literal>/home/alice/nixpkgs/default.nix</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--strict</option></term>
-
-    <listitem><para>When used with <option>--eval</option>,
-    recursively evaluate list elements and attributes.  Normally, such
-    sub-expressions are left unevaluated (since the Nix expression
-    language is lazy).</para>
-
-    <warning><para>This option can cause non-termination, because lazy
-    data structures can be infinitely large.</para></warning>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--json</option></term>
-
-    <listitem><para>When used with <option>--eval</option>, print the resulting
-    value as an JSON representation of the abstract syntax tree rather
-    than as an ATerm.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--xml</option></term>
-
-    <listitem><para>When used with <option>--eval</option>, print the resulting
-    value as an XML representation of the abstract syntax tree rather than as
-    an ATerm. The schema is the same as that used by the <link
-    linkend="builtin-toXML"><function>toXML</function> built-in</link>.
-    </para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--read-write-mode</option></term>
-
-    <listitem><para>When used with <option>--eval</option>, perform
-    evaluation in read/write mode so nix language features that
-    require it will still work (at the cost of needing to do
-    instantiation of every evaluated derivation). If this option is
-    not enabled, there may be uninstantiated store paths in the final
-    output.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<variablelist condition="manpage">
-  <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>Instantiating store derivations from a Nix expression, and
-building them using <command>nix-store</command>:
-
-<screen>
-$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
-/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
-
-$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
-<emphasis>...</emphasis>
-/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
-
-$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
-dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
-...</screen>
-
-</para>
-
-<para>You can also give a Nix expression on the command line:
-
-<screen>
-$ nix-instantiate -E 'with import &lt;nixpkgs> { }; hello'
-/nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv
-</screen>
-
-This is equivalent to:
-
-<screen>
-$ nix-instantiate '&lt;nixpkgs>' -A hello
-</screen>
-
-</para>
-
-<para>Parsing and evaluating Nix expressions:
-
-<screen>
-$ nix-instantiate --parse -E '1 + 2'
-1 + 2
-
-$ nix-instantiate --eval -E '1 + 2'
-3
-
-$ nix-instantiate --eval --xml -E '1 + 2'
-<![CDATA[<?xml version='1.0' encoding='utf-8'?>
-<expr>
-  <int value="3" />
-</expr>]]></screen>
-
-</para>
-
-<para>The difference between non-strict and strict evaluation:
-
-<screen>
-$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }'
-<emphasis>...</emphasis><![CDATA[
-  <attr name="x">
-    <string value="foo" />
-  </attr>
-  <attr name="y">
-    <unevaluated />
-  </attr>]]>
-<emphasis>...</emphasis></screen>
-
-Note that <varname>y</varname> is left unevaluated (the XML
-representation doesn’t attempt to show non-normal forms).
-
-<screen>
-$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }'
-<emphasis>...</emphasis><![CDATA[
-  <attr name="x">
-    <string value="foo" />
-  </attr>
-  <attr name="y">
-    <string value="foo" />
-  </attr>]]>
-<emphasis>...</emphasis></screen>
-
-</para>
-
-</refsection>
-
-
-<refsection condition="manpage"><title>Environment variables</title>
-
-<variablelist>
-  <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml
deleted file mode 100644
index db2a6960a..000000000
--- a/doc/manual/command-ref/nix-prefetch-url.xml
+++ /dev/null
@@ -1,131 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-prefetch-url">
-
-<refmeta>
-  <refentrytitle>nix-prefetch-url</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-prefetch-url</refname>
-  <refpurpose>copy a file from a URL into the store and print its hash</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-prefetch-url</command>
-    <arg><option>--version</option></arg>
-    <arg><option>--type</option> <emphasis>hashAlgo</emphasis></arg>
-    <arg><option>--print-path</option></arg>
-    <arg><option>--unpack</option></arg>
-    <arg><option>--name</option> <emphasis>name</emphasis></arg>
-    <arg choice='plain'><emphasis>url</emphasis></arg>
-    <arg><emphasis>hash</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-prefetch-url</command> downloads the
-file referenced by the URL <emphasis>url</emphasis>, prints its
-cryptographic hash, and copies it into the Nix store.  The file name
-in the store is
-<filename><emphasis>hash</emphasis>-<emphasis>baseName</emphasis></filename>,
-where <emphasis>baseName</emphasis> is everything following the
-final slash in <emphasis>url</emphasis>.</para>
-
-<para>This command is just a convenience for Nix expression writers.
-Often a Nix expression fetches some source distribution from the
-network using the <literal>fetchurl</literal> expression contained in
-Nixpkgs.  However, <literal>fetchurl</literal> requires a
-cryptographic hash.  If you don't know the hash, you would have to
-download the file first, and then <literal>fetchurl</literal> would
-download it again when you build your Nix expression.  Since
-<literal>fetchurl</literal> uses the same name for the downloaded file
-as <command>nix-prefetch-url</command>, the redundant download can be
-avoided.</para>
-
-<para>If <emphasis>hash</emphasis> is specified, then a download
-is not performed if the Nix store already contains a file with the
-same hash and base name.  Otherwise, the file is downloaded, and an
-error is signaled if the actual hash of the file does not match the
-specified hash.</para>
-
-<para>This command prints the hash on standard output.  Additionally,
-if the option <option>--print-path</option> is used, the path of the
-downloaded file in the Nix store is also printed.</para>
-
-</refsection>
-
-
-<refsection><title>Options</title>
-
-<variablelist>
-
-  <varlistentry><term><option>--type</option> <emphasis>hashAlgo</emphasis></term>
-
-    <listitem><para>Use the specified cryptographic hash algorithm,
-    which can be one of <literal>md5</literal>,
-    <literal>sha1</literal>, and
-    <literal>sha256</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--print-path</option></term>
-
-    <listitem><para>Print the store path of the downloaded file on
-    standard output.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--unpack</option></term>
-
-    <listitem><para>Unpack the archive (which must be a tarball or zip
-    file) and add the result to the Nix store. The resulting hash can
-    be used with functions such as Nixpkgs’s
-    <varname>fetchzip</varname> or
-    <varname>fetchFromGitHub</varname>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--name</option> <emphasis>name</emphasis></term>
-
-    <listitem><para>Override the name of the file in the Nix store. By
-    default, this is
-    <literal><emphasis>hash</emphasis>-<emphasis>basename</emphasis></literal>,
-    where <emphasis>basename</emphasis> is the last component of
-    <emphasis>url</emphasis>. Overriding the name is necessary
-    when <emphasis>basename</emphasis> contains characters that
-    are not allowed in Nix store paths.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
-0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
-
-$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
-0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
-/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
-
-$ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz
-079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7
-/nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz
-</screen>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
deleted file mode 100644
index 1d55b5622..000000000
--- a/doc/manual/command-ref/nix-shell.xml
+++ /dev/null
@@ -1,411 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-shell">
-
-<refmeta>
-  <refentrytitle>nix-shell</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-shell</refname>
-  <refpurpose>start an interactive shell based on a Nix expression</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-shell</command>
-    <arg><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></arg>
-    <arg>
-      <group choice='req'>
-        <arg choice='plain'><option>--attr</option></arg>
-        <arg choice='plain'><option>-A</option></arg>
-      </group>
-      <emphasis>attrPath</emphasis>
-    </arg>
-    <arg><option>--command</option> <emphasis>cmd</emphasis></arg>
-    <arg><option>--run</option> <emphasis>cmd</emphasis></arg>
-    <arg><option>--exclude</option> <emphasis>regexp</emphasis></arg>
-    <arg><option>--pure</option></arg>
-    <arg><option>--keep</option> <emphasis>name</emphasis></arg>
-    <group choice='req'>
-      <arg choice='plain'>
-        <group choice='req'>
-          <arg choice='plain'><option>--packages</option></arg>
-          <arg choice='plain'><option>-p</option></arg>
-        </group>
-        <arg choice='plain' rep='repeat'>
-          <group choice='req'>
-            <arg choice="plain"><emphasis>packages</emphasis></arg>
-            <arg choice="plain"><emphasis>expressions</emphasis></arg>
-          </group>
-        </arg>
-      </arg>
-      <arg><emphasis>path</emphasis></arg>
-    </group>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-shell</command> will build the
-dependencies of the specified derivation, but not the derivation
-itself.  It will then start an interactive shell in which all
-environment variables defined by the derivation
-<emphasis>path</emphasis> have been set to their corresponding
-values, and the script <literal>$stdenv/setup</literal> has been
-sourced.  This is useful for reproducing the environment of a
-derivation for development.</para>
-
-<para>If <emphasis>path</emphasis> is not given,
-<command>nix-shell</command> defaults to
-<filename>shell.nix</filename> if it exists, and
-<filename>default.nix</filename> otherwise.</para>
-
-<para>If <emphasis>path</emphasis> starts with
-<literal>http://</literal> or <literal>https://</literal>, it is
-interpreted as the URL of a tarball that will be downloaded and
-unpacked to a temporary location. The tarball must include a single
-top-level directory containing at least a file named
-<filename>default.nix</filename>.</para>
-
-<para>If the derivation defines the variable
-<varname>shellHook</varname>, it will be evaluated after
-<literal>$stdenv/setup</literal> has been sourced.  Since this hook is
-not executed by regular Nix builds, it allows you to perform
-initialisation specific to <command>nix-shell</command>.  For example,
-the derivation attribute
-
-<programlisting>
-shellHook =
-  ''
-    echo "Hello shell"
-  '';
-</programlisting>
-
-will cause <command>nix-shell</command> to print <literal>Hello shell</literal>.</para>
-
-</refsection>
-
-
-<refsection><title>Options</title>
-
-<para>All options not listed here are passed to <command>nix-store
---realise</command>, except for <option>--arg</option> and
-<option>--attr</option> / <option>-A</option> which are passed to
-<command>nix-instantiate</command>.  <phrase condition="manual">See
-also <xref linkend="sec-common-options" />.</phrase></para>
-
-<variablelist>
-
-  <varlistentry><term><option>--command</option> <emphasis>cmd</emphasis></term>
-
-    <listitem><para>In the environment of the derivation, run the
-    shell command <emphasis>cmd</emphasis>. This command is
-    executed in an interactive shell. (Use <option>--run</option> to
-    use a non-interactive shell instead.) However, a call to
-    <literal>exit</literal> is implicitly added to the command, so the
-    shell will exit after running the command. To prevent this, add
-    <literal>return</literal> at the end; e.g. <literal>--command
-    "echo Hello; return"</literal> will print <literal>Hello</literal>
-    and then drop you into the interactive shell. This can be useful
-    for doing any additional initialisation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--run</option> <emphasis>cmd</emphasis></term>
-
-    <listitem><para>Like <option>--command</option>, but executes the
-    command in a non-interactive shell. This means (among other
-    things) that if you hit Ctrl-C while the command is running, the
-    shell exits.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--exclude</option> <emphasis>regexp</emphasis></term>
-
-    <listitem><para>Do not build any dependencies whose store path
-    matches the regular expression <emphasis>regexp</emphasis>.
-    This option may be specified multiple times.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--pure</option></term>
-
-    <listitem><para>If this flag is specified, the environment is
-    almost entirely cleared before the interactive shell is started,
-    so you get an environment that more closely corresponds to the
-    “real” Nix build.  A few variables, in particular
-    <literal>HOME</literal>, <literal>USER</literal> and
-    <literal>DISPLAY</literal>, are retained.  Note that
-    <filename>~/.bashrc</filename> and (depending on your Bash
-    installation) <filename>/etc/bashrc</filename> are still sourced,
-    so any variables set there will affect the interactive
-    shell.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--packages</option> / <option>-p</option> <emphasis>packages</emphasis>…</term>
-
-    <listitem><para>Set up an environment in which the specified
-    packages are present.  The command line arguments are interpreted
-    as attribute names inside the Nix Packages collection.  Thus,
-    <literal>nix-shell -p libjpeg openjdk</literal> will start a shell
-    in which the packages denoted by the attribute names
-    <varname>libjpeg</varname> and <varname>openjdk</varname> are
-    present.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>-i</option> <emphasis>interpreter</emphasis></term>
-
-    <listitem><para>The chained script interpreter to be invoked by
-    <command>nix-shell</command>. Only applicable in
-    <literal>#!</literal>-scripts (described <link
-    linkend="ssec-nix-shell-shebang">below</link>).</para>
-
-    </listitem></varlistentry>
-
-  <varlistentry><term><option>--keep</option> <emphasis>name</emphasis></term>
-
-    <listitem><para>When a <option>--pure</option> shell is started,
-    keep the listed environment variables.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<para>The following common options are supported:</para>
-
-<variablelist condition="manpage">
-  <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Environment variables</title>
-
-<variablelist>
-
-  <varlistentry><term><literal>NIX_BUILD_SHELL</literal></term>
-
-    <listitem><para>Shell used to start the interactive environment.
-    Defaults to the <command>bash</command> found in <literal>PATH</literal>.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To build the dependencies of the package Pan, and start an
-interactive shell in which to build it:
-
-<screen>
-$ nix-shell '&lt;nixpkgs>' -A pan
-[nix-shell]$ unpackPhase
-[nix-shell]$ cd pan-*
-[nix-shell]$ configurePhase
-[nix-shell]$ buildPhase
-[nix-shell]$ ./pan/gui/pan
-</screen>
-
-To clear the environment first, and do some additional automatic
-initialisation of the interactive shell:
-
-<screen>
-$ nix-shell '&lt;nixpkgs>' -A pan --pure \
-    --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
-</screen>
-
-Nix expressions can also be given on the command line using the
-<command>-E</command> and <command>-p</command> flags.
-For instance, the following starts a shell containing the packages
-<literal>sqlite</literal> and <literal>libX11</literal>:
-
-<screen>
-$ nix-shell -E 'with import &lt;nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
-</screen>
-
-A shorter way to do the same is:
-
-<screen>
-$ nix-shell -p sqlite xorg.libX11
-[nix-shell]$ echo $NIX_LDFLAGS
-… -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
-</screen>
-
-Note that <command>-p</command> accepts multiple full nix expressions that
-are valid in the <literal>buildInputs = [ ... ]</literal> shown above,
-not only package names. So the following is also legal:
-
-<screen>
-$ nix-shell -p sqlite 'git.override { withManual = false; }'
-</screen>
-
-The <command>-p</command> flag looks up Nixpkgs in the Nix search
-path. You can override it by passing <option>-I</option> or setting
-<literal>NIX_PATH</literal>. For example, the following gives you a shell
-containing the Pan package from a specific revision of Nixpkgs:
-
-<screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
-
-[nix-shell:~]$ pan --version
-Pan 0.139
-</screen>
-
-</para>
-
-</refsection>
-
-
-<refsection xml:id="ssec-nix-shell-shebang"><title>Use as a <literal>#!</literal>-interpreter</title>
-
-<para>You can use <command>nix-shell</command> as a script interpreter
-to allow scripts written in arbitrary languages to obtain their own
-dependencies via Nix. This is done by starting the script with the
-following lines:
-
-<programlisting>
-#! /usr/bin/env nix-shell
-#! nix-shell -i <emphasis>real-interpreter</emphasis> -p <emphasis>packages</emphasis>
-</programlisting>
-
-where <emphasis>real-interpreter</emphasis> is the “real” script
-interpreter that will be invoked by <command>nix-shell</command> after
-it has obtained the dependencies and initialised the environment, and
-<emphasis>packages</emphasis> are the attribute names of the
-dependencies in Nixpkgs.</para>
-
-<para>The lines starting with <literal>#! nix-shell</literal> specify
-<command>nix-shell</command> options (see above). Note that you cannot
-write <literal>#! /usr/bin/env nix-shell -i ...</literal> because
-many operating systems only allow one argument in
-<literal>#!</literal> lines.</para>
-
-<para>For example, here is a Python script that depends on Python and
-the <literal>prettytable</literal> package:
-
-<programlisting>
-#! /usr/bin/env nix-shell
-#! nix-shell -i python -p python pythonPackages.prettytable
-
-import prettytable
-
-# Print a simple table.
-t = prettytable.PrettyTable(["N", "N^2"])
-for n in range(1, 10): t.add_row([n, n * n])
-print t
-</programlisting>
-
-</para>
-
-<para>Similarly, the following is a Perl script that specifies that it
-requires Perl and the <literal>HTML::TokeParser::Simple</literal> and
-<literal>LWP</literal> packages:
-
-<programlisting>
-#! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
-
-use HTML::TokeParser::Simple;
-
-# Fetch nixos.org and print all hrefs.
-my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
-
-while (my $token = $p->get_tag("a")) {
-    my $href = $token->get_attr("href");
-    print "$href\n" if $href;
-}
-</programlisting>
-
-</para>
-
-<para>Sometimes you need to pass a simple Nix expression to customize
-a package like Terraform:
-
-<programlisting><![CDATA[
-#! /usr/bin/env nix-shell
-#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
-
-terraform apply
-]]></programlisting>
-
-<note><para>You must use double quotes (<literal>"</literal>) when
-passing a simple Nix expression in a nix-shell shebang.</para></note>
-</para>
-
-<para>Finally, using the merging of multiple nix-shell shebangs the
-following Haskell script uses a specific branch of Nixpkgs/NixOS (the
-18.03 stable branch):
-
-<programlisting><![CDATA[
-#! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
-
-import Network.HTTP
-import Text.HTML.TagSoup
-
--- Fetch nixos.org and print all hrefs.
-main = do
-  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
-  body <- getResponseBody resp
-  let tags = filter (isTagOpenName "a") $ parseTags body
-  let tags' = map (fromAttrib "href") tags
-  mapM_ putStrLn $ filter (/= "") tags'
-]]></programlisting>
-
-If you want to be even more precise, you can specify a specific
-revision of Nixpkgs:
-
-<programlisting>
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
-</programlisting>
-
-</para>
-
-<para>The examples above all used <option>-p</option> to get
-dependencies from Nixpkgs. You can also use a Nix expression to build
-your own dependencies. For example, the Python example could have been
-written as:
-
-<programlisting>
-#! /usr/bin/env nix-shell
-#! nix-shell deps.nix -i python
-</programlisting>
-
-where the file <filename>deps.nix</filename> in the same directory
-as the <literal>#!</literal>-script contains:
-
-<programlisting>
-with import &lt;nixpkgs> {};
-
-runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
-</programlisting>
-
-</para>
-
-</refsection>
-
-
-<refsection condition="manpage"><title>Environment variables</title>
-
-<variablelist>
-  <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
deleted file mode 100644
index 352e716ae..000000000
--- a/doc/manual/command-ref/nix-store.xml
+++ /dev/null
@@ -1,1516 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-nix-store">
-
-<refmeta>
-  <refentrytitle>nix-store</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-store</refname>
-  <refpurpose>manipulate or query the Nix store</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
-    <arg><option>--add-root</option> <emphasis>path</emphasis></arg>
-    <arg><option>--indirect</option></arg>
-    <arg choice='plain'><emphasis>operation</emphasis></arg>
-    <arg rep='repeat'><emphasis>options</emphasis></arg>
-    <arg rep='repeat'><emphasis>arguments</emphasis></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-store</command> performs primitive
-operations on the Nix store.  You generally do not need to run this
-command manually.</para>
-
-<para><command>nix-store</command> takes exactly one
-<emphasis>operation</emphasis> flag which indicates the subcommand to
-be performed.  These are documented below.</para>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Common options</title>
-
-<para>This section lists the options that are common to all
-operations.  These options are allowed for every subcommand, though
-they may not always have an effect.  <phrase condition="manual">See
-also <xref linkend="sec-common-options" /> for a list of common
-options.</phrase></para>
-
-<variablelist>
-
-  <varlistentry xml:id="opt-add-root"><term><option>--add-root</option> <emphasis>path</emphasis></term>
-
-    <listitem><para>Causes the result of a realisation
-    (<option>--realise</option> and <option>--force-realise</option>)
-    to be registered as a root of the garbage collector<phrase
-    condition="manual"> (see <xref linkend="ssec-gc-roots"
-    />)</phrase>.  The root is stored in
-    <emphasis>path</emphasis>, which must be inside a directory
-    that is scanned for roots by the garbage collector (i.e.,
-    typically in a subdirectory of
-    <filename>/nix/var/nix/gcroots/</filename>)
-    <emphasis>unless</emphasis> the <option>--indirect</option> flag
-    is used.</para>
-
-    <para>If there are multiple results, then multiple symlinks will
-    be created by sequentially numbering symlinks beyond the first one
-    (e.g., <filename>foo</filename>, <filename>foo-2</filename>,
-    <filename>foo-3</filename>, and so on).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--indirect</option></term>
-
-    <listitem>
-
-    <para>In conjunction with <option>--add-root</option>, this option
-    allows roots to be stored <emphasis>outside</emphasis> of the GC
-    roots directory.  This is useful for commands such as
-    <command>nix-build</command> that place a symlink to the build
-    result in the current directory; such a build result should not be
-    garbage-collected unless the symlink is removed.</para>
-
-    <para>The <option>--indirect</option> flag causes a uniquely named
-    symlink to <emphasis>path</emphasis> to be stored in
-    <filename>/nix/var/nix/gcroots/auto/</filename>.  For instance,
-
-    <screen>
-$ nix-store --add-root /home/eelco/bla/result --indirect -r <emphasis>...</emphasis>
-
-$ ls -l /nix/var/nix/gcroots/auto
-lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
-
-$ ls -l /home/eelco/bla/result
-lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10</screen>
-
-    Thus, when <filename>/home/eelco/bla/result</filename> is removed,
-    the GC root in the <filename>auto</filename> directory becomes a
-    dangling symlink and will be ignored by the collector.</para>
-
-    <warning><para>Note that it is not possible to move or rename
-    indirect GC roots, since the symlink in the
-    <filename>auto</filename> directory will still point to the old
-    location.</para></warning>
-
-    </listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<variablelist condition="manpage">
-  <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id='rsec-nix-store-realise'><title>Operation <option>--realise</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--realise</option></arg>
-    <arg choice='plain'><option>-r</option></arg>
-  </group>
-  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-  <arg><option>--dry-run</option></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--realise</option> essentially “builds”
-the specified store paths.  Realisation is a somewhat overloaded term:
-
-<itemizedlist>
-
-  <listitem><para>If the store path is a
-  <emphasis>derivation</emphasis>, realisation ensures that the output
-  paths of the derivation are <link
-  linkend="gloss-validity">valid</link> (i.e., the output path and its
-  closure exist in the file system).  This can be done in several
-  ways.  First, it is possible that the outputs are already valid, in
-  which case we are done immediately.  Otherwise, there may be <link
-  linkend="gloss-substitute">substitutes</link> that produce the
-  outputs (e.g., by downloading them).  Finally, the outputs can be
-  produced by performing the build action described by the
-  derivation.</para></listitem>
-
-  <listitem><para>If the store path is not a derivation, realisation
-  ensures that the specified path is valid (i.e., it and its closure
-  exist in the file system).  If the path is already valid, we are
-  done immediately.  Otherwise, the path and any missing paths in its
-  closure may be produced through substitutes.  If there are no
-  (successful) subsitutes, realisation fails.</para></listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>The output path of each derivation is printed on standard
-output.  (For non-derivations argument, the argument itself is
-printed.)</para>
-
-<para>The following flags are available:</para>
-
-<variablelist>
-
-  <varlistentry><term><option>--dry-run</option></term>
-
-    <listitem><para>Print on standard error a description of what
-    packages would be built or downloaded, without actually performing
-    the operation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--ignore-unknown</option></term>
-
-    <listitem><para>If a non-derivation path does not have a
-    substitute, then silently ignore it.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--check</option></term>
-
-    <listitem><para>This option allows you to check whether a
-    derivation is deterministic. It rebuilds the specified derivation
-    and checks whether the result is bitwise-identical with the
-    existing outputs, printing an error if that’s not the case. The
-    outputs of the specified derivation must already exist. When used
-    with <option>-K</option>, if an output path is not identical to
-    the corresponding output from the previous build, the new output
-    path is left in
-    <filename>/nix/store/<emphasis>name</emphasis>.check.</filename></para>
-
-    <para>See also the <option>build-repeat</option> configuration
-    option, which repeats a derivation a number of times and prevents
-    its outputs from being registered as “valid” in the Nix store
-    unless they are identical.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<para>Special exit codes:</para>
-
-<variablelist>
-
-  <varlistentry><term><literal>100</literal></term>
-    <listitem><para>Generic build failure, the builder process
-    returned with a non-zero exit code.</para></listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>101</literal></term>
-    <listitem><para>Build timeout, the build was aborted because it
-    did not complete within the specified <link
-    linkend='conf-timeout'><literal>timeout</literal></link>.
-    </para></listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>102</literal></term>
-    <listitem><para>Hash mismatch, the build output was rejected
-    because it does not match the specified <link
-    linkend="fixed-output-drvs"><varname>outputHash</varname></link>.
-    </para></listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>104</literal></term>
-    <listitem><para>Not deterministic, the build succeeded in check
-    mode but the resulting output is not binary reproducable.</para>
-    </listitem>
-  </varlistentry>
-
-</variablelist>
-
-<para>With the <option>--keep-going</option> flag it's possible for
-multiple failures to occur, in this case the 1xx status codes are or combined
-using binary or. <screen>
-1100100
-   ^^^^
-   |||`- timeout
-   ||`-- output hash mismatch
-   |`--- build failure
-   `---- not deterministic
-</screen></para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>This operation is typically used to build store derivations
-produced by <link
-linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>:
-
-<screen>
-$ nix-store -r $(nix-instantiate ./test.nix)
-/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1</screen>
-
-This is essentially what <link
-linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
-
-<para>To test whether a previously-built derivation is deterministic:
-
-<screen>
-$ nix-build '&lt;nixpkgs>' -A hello --check -K
-</screen>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id='rsec-nix-store-serve'><title>Operation <option>--serve</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>--serve</option></arg>
-  <arg><option>--write</option></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--serve</option> provides access to
-the Nix store over stdin and stdout, and is intended to be used
-as a means of providing Nix store access to a restricted ssh user.
-</para>
-
-<para>The following flags are available:</para>
-
-<variablelist>
-
-  <varlistentry><term><option>--write</option></term>
-
-    <listitem><para>Allow the connected client to request the realization
-    of derivations. In effect, this can be used to make the host act
-    as a remote builder.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To turn a host into a build server, the
-<filename>authorized_keys</filename> file can be used to provide build
-access to a given SSH public key:
-
-<screen>
-$ cat &lt;&lt;EOF >>/root/.ssh/authorized_keys
-command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA...
-EOF
-</screen>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id='rsec-nix-store-gc'><title>Operation <option>--gc</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>--gc</option></arg>
-  <group>
-    <arg choice='plain'><option>--print-roots</option></arg>
-    <arg choice='plain'><option>--print-live</option></arg>
-    <arg choice='plain'><option>--print-dead</option></arg>
-  </group>
-  <arg><option>--max-freed</option> <emphasis>bytes</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>Without additional flags, the operation <option>--gc</option>
-performs a garbage collection on the Nix store.  That is, all paths in
-the Nix store not reachable via file system references from a set of
-“roots”, are deleted.</para>
-
-<para>The following suboperations may be specified:</para>
-
-<variablelist>
-
-  <varlistentry><term><option>--print-roots</option></term>
-
-    <listitem><para>This operation prints on standard output the set
-    of roots used by the garbage collector.  What constitutes a root
-    is described in <xref linkend="ssec-gc-roots"
-    />.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--print-live</option></term>
-
-    <listitem><para>This operation prints on standard output the set
-    of “live” store paths, which are all the store paths reachable
-    from the roots.  Live paths should never be deleted, since that
-    would break consistency — it would become possible that
-    applications are installed that reference things that are no
-    longer present in the store.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--print-dead</option></term>
-
-    <listitem><para>This operation prints out on standard output the
-    set of “dead” store paths, which is just the opposite of the set
-    of live paths: any path in the store that is not live (with
-    respect to the roots) is dead.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-<para>By default, all unreachable paths are deleted.  The following
-options control what gets deleted and in what order:
-
-<variablelist>
-
-  <varlistentry><term><option>--max-freed</option> <emphasis>bytes</emphasis></term>
-
-    <listitem><para>Keep deleting paths until at least
-    <emphasis>bytes</emphasis> bytes have been deleted, then
-    stop.  The argument <emphasis>bytes</emphasis> can be
-    followed by the multiplicative suffix <literal>K</literal>,
-    <literal>M</literal>, <literal>G</literal> or
-    <literal>T</literal>, denoting KiB, MiB, GiB or TiB
-    units.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</para>
-
-<para>The behaviour of the collector is also influenced by the <link
-linkend="conf-keep-outputs"><literal>keep-outputs</literal></link>
-and <link
-linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
-variables in the Nix configuration file.</para>
-
-<para>By default, the collector prints the total number of freed bytes
-when it finishes (or when it is interrupted). With
-<option>--print-dead</option>, it prints the number of bytes that would
-be freed.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To delete all unreachable paths, just do:
-
-<screen>
-$ nix-store --gc
-deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
-<emphasis>...</emphasis>
-8825586 bytes freed (8.42 MiB)</screen>
-
-</para>
-
-<para>To delete at least 100 MiBs of unreachable paths:
-
-<screen>
-$ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--delete</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>--delete</option></arg>
-  <arg><option>--ignore-liveness</option></arg>
-  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--delete</option> deletes the store paths
-<emphasis>paths</emphasis> from the Nix store, but only if it is
-safe to do so; that is, when the path is not reachable from a root of
-the garbage collector.  This means that you can only delete paths that
-would also be deleted by <literal>nix-store --gc</literal>.  Thus,
-<literal>--delete</literal> is a more targeted version of
-<literal>--gc</literal>.</para>
-
-<para>With the option <option>--ignore-liveness</option>, reachability
-from the roots is ignored.  However, the path still won’t be deleted
-if there are other paths in the store that refer to it (i.e., depend
-on it).</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4
-0 bytes freed (0.00 MiB)
-error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive</screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id='refsec-nix-store-query'><title>Operation <option>--query</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--query</option></arg>
-    <arg choice='plain'><option>-q</option></arg>
-  </group>
-  <group choice='req'>
-    <arg choice='plain'><option>--outputs</option></arg>
-    <arg choice='plain'><option>--requisites</option></arg>
-    <arg choice='plain'><option>-R</option></arg>
-    <arg choice='plain'><option>--references</option></arg>
-    <arg choice='plain'><option>--referrers</option></arg>
-    <arg choice='plain'><option>--referrers-closure</option></arg>
-    <arg choice='plain'><option>--deriver</option></arg>
-    <arg choice='plain'><option>-d</option></arg>
-    <arg choice='plain'><option>--graph</option></arg>
-    <arg choice='plain'><option>--tree</option></arg>
-    <arg choice='plain'><option>--binding</option> <emphasis>name</emphasis></arg>
-    <arg choice='plain'><option>-b</option> <emphasis>name</emphasis></arg>
-    <arg choice='plain'><option>--hash</option></arg>
-    <arg choice='plain'><option>--size</option></arg>
-    <arg choice='plain'><option>--roots</option></arg>
-  </group>
-  <arg><option>--use-output</option></arg>
-  <arg><option>-u</option></arg>
-  <arg><option>--force-realise</option></arg>
-  <arg><option>-f</option></arg>
-  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--query</option> displays various bits of
-information about the store paths .  The queries are described below.  At
-most one query can be specified.  The default query is
-<option>--outputs</option>.</para>
-
-<para>The paths <emphasis>paths</emphasis> may also be symlinks
-from outside of the Nix store, to the Nix store.  In that case, the
-query is applied to the target of the symlink.</para>
-
-
-</refsection>
-
-
-<refsection><title>Common query options</title>
-
-<variablelist>
-
-  <varlistentry><term><option>--use-output</option></term>
-    <term><option>-u</option></term>
-
-    <listitem><para>For each argument to the query that is a store
-    derivation, apply the query to the output path of the derivation
-    instead.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--force-realise</option></term>
-    <term><option>-f</option></term>
-
-    <listitem><para>Realise each argument to the query first (see
-    <link linkend="rsec-nix-store-realise"><command>nix-store
-    --realise</command></link>).</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection xml:id='nixref-queries'><title>Queries</title>
-
-<variablelist>
-
-  <varlistentry><term><option>--outputs</option></term>
-
-    <listitem><para>Prints out the <link
-    linkend="gloss-output-path">output paths</link> of the store
-    derivations <emphasis>paths</emphasis>.  These are the paths
-    that will be produced when the derivation is
-    built.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--requisites</option></term>
-    <term><option>-R</option></term>
-
-    <listitem><para>Prints out the <link
-    linkend="gloss-closure">closure</link> of the store path
-    <emphasis>paths</emphasis>.</para>
-
-    <para>This query has one option:</para>
-
-    <variablelist>
-
-      <varlistentry><term><option>--include-outputs</option></term>
-
-        <listitem><para>Also include the output path of store
-        derivations, and their closures.</para></listitem>
-
-      </varlistentry>
-
-    </variablelist>
-
-    <para>This query can be used to implement various kinds of
-    deployment.  A <emphasis>source deployment</emphasis> is obtained
-    by distributing the closure of a store derivation.  A
-    <emphasis>binary deployment</emphasis> is obtained by distributing
-    the closure of an output path.  A <emphasis>cache
-    deployment</emphasis> (combined source/binary deployment,
-    including binaries of build-time-only dependencies) is obtained by
-    distributing the closure of a store derivation and specifying the
-    option <option>--include-outputs</option>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--references</option></term>
-
-    <listitem><para>Prints the set of <link
-    linkend="gloss-reference">references</link> of the store paths
-    <emphasis>paths</emphasis>, that is, their immediate
-    dependencies.  (For <emphasis>all</emphasis> dependencies, use
-    <option>--requisites</option>.)</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--referrers</option></term>
-
-    <listitem><para>Prints the set of <emphasis>referrers</emphasis> of
-    the store paths <emphasis>paths</emphasis>, that is, the
-    store paths currently existing in the Nix store that refer to one
-    of <emphasis>paths</emphasis>.  Note that contrary to the
-    references, the set of referrers is not constant; it can change as
-    store paths are added or removed.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--referrers-closure</option></term>
-
-    <listitem><para>Prints the closure of the set of store paths
-    <emphasis>paths</emphasis> under the referrers relation; that
-    is, all store paths that directly or indirectly refer to one of
-    <emphasis>paths</emphasis>.  These are all the path currently
-    in the Nix store that are dependent on
-    <emphasis>paths</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--deriver</option></term>
-    <term><option>-d</option></term>
-
-    <listitem><para>Prints the <link
-    linkend="gloss-deriver">deriver</link> of the store paths
-    <emphasis>paths</emphasis>.  If the path has no deriver
-    (e.g., if it is a source file), or if the deriver is not known
-    (e.g., in the case of a binary-only deployment), the string
-    <literal>unknown-deriver</literal> is printed.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--graph</option></term>
-
-    <listitem><para>Prints the references graph of the store paths
-    <emphasis>paths</emphasis> in the format of the
-    <command>dot</command> tool of AT&amp;T's <link
-    xlink:href="http://www.graphviz.org/">Graphviz package</link>.
-    This can be used to visualise dependency graphs.  To obtain a
-    build-time dependency graph, apply this to a store derivation.  To
-    obtain a runtime dependency graph, apply it to an output
-    path.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--tree</option></term>
-
-    <listitem><para>Prints the references graph of the store paths
-    <emphasis>paths</emphasis> as a nested ASCII tree.
-    References are ordered by descending closure size; this tends to
-    flatten the tree, making it more readable.  The query only
-    recurses into a store path when it is first encountered; this
-    prevents a blowup of the tree representation of the
-    graph.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--graphml</option></term>
-
-    <listitem><para>Prints the references graph of the store paths
-    <emphasis>paths</emphasis> in the <link
-    xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format.
-    This can be used to visualise dependency graphs. To obtain a
-    build-time dependency graph, apply this to a store derivation. To
-    obtain a runtime dependency graph, apply it to an output
-    path.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--binding</option> <emphasis>name</emphasis></term>
-    <term><option>-b</option> <emphasis>name</emphasis></term>
-
-    <listitem><para>Prints the value of the attribute
-    <emphasis>name</emphasis> (i.e., environment variable) of
-    the store derivations <emphasis>paths</emphasis>.  It is an
-    error for a derivation to not have the specified
-    attribute.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--hash</option></term>
-
-    <listitem><para>Prints the SHA-256 hash of the contents of the
-    store paths <emphasis>paths</emphasis> (that is, the hash of
-    the output of <command>nix-store --dump</command> on the given
-    paths).  Since the hash is stored in the Nix database, this is a
-    fast operation.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--size</option></term>
-
-    <listitem><para>Prints the size in bytes of the contents of the
-    store paths <emphasis>paths</emphasis> — to be precise, the
-    size of the output of <command>nix-store --dump</command> on the
-    given paths.  Note that the actual disk space required by the
-    store paths may be higher, especially on filesystems with large
-    cluster sizes.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--roots</option></term>
-
-    <listitem><para>Prints the garbage collector roots that point,
-    directly or indirectly, at the store paths
-    <emphasis>paths</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>Print the closure (runtime dependencies) of the
-<command>svn</command> program in the current user environment:
-
-<screen>
-$ nix-store -qR $(which svn)
-/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
-/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
-<emphasis>...</emphasis></screen>
-
-</para>
-
-<para>Print the build-time dependencies of <command>svn</command>:
-
-<screen>
-$ nix-store -qR $(nix-store -qd $(which svn))
-/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
-/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
-/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
-<emphasis>... lots of other paths ...</emphasis></screen>
-
-The difference with the previous example is that we ask the closure of
-the derivation (<option>-qd</option>), not the closure of the output
-path that contains <command>svn</command>.</para>
-
-<para>Show the build-time dependencies as a tree:
-
-<screen>
-$ nix-store -q --tree $(nix-store -qd $(which svn))
-/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
-+---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
-+---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
-|   +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
-|   +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
-<emphasis>...</emphasis></screen>
-
-</para>
-
-<para>Show all paths that depend on the same OpenSSL library as
-<command>svn</command>:
-
-<screen>
-$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn)))
-/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0
-/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
-/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3
-/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5</screen>
-
-</para>
-
-<para>Show all paths that directly or indirectly depend on the Glibc
-(C library) used by <command>svn</command>:
-
-<screen>
-$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
-/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
-/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
-<emphasis>...</emphasis></screen>
-
-Note that <command>ldd</command> is a command that prints out the
-dynamic libraries used by an ELF executable.</para>
-
-<para>Make a picture of the runtime dependency graph of the current
-user environment:
-
-<screen>
-$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps
-$ gv graph.ps</screen>
-
-</para>
-
-<para>Show every garbage collector root that points to a store path
-that depends on <command>svn</command>:
-
-<screen>
-$ nix-store -q --roots $(which svn)
-/nix/var/nix/profiles/default-81-link
-/nix/var/nix/profiles/default-82-link
-/nix/var/nix/profiles/per-user/eelco/profile-97-link
-</screen>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<!--
-<refsection xml:id="rsec-nix-store-reg-val"><title>Operation <option>-XXX-register-validity</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>-XXX-register-validity</option></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>TODO</para>
-
-</refsection>
-
-</refsection>
--->
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--add</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>--add</option></arg>
-  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--add</option> adds the specified paths to
-the Nix store.  It prints the resulting paths in the Nix store on
-standard output.</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store --add ./foo.c
-/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c</screen>
-
-</refsection>
-
-</refsection>
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--add-fixed</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg><option>--recursive</option></arg>
-  <arg choice='plain'><option>--add-fixed</option></arg>
-  <arg choice='plain'><emphasis>algorithm</emphasis></arg>
-  <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-</cmdsynopsis>
-
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--add-fixed</option> adds the specified paths to
-the Nix store.  Unlike <option>--add</option> paths are registered using the
-specified hashing algorithm, resulting in the same output path as a fixed-output
-derivation.  This can be used for sources that are not available from a public
-url or broke since the download expression was written.
-</para>
-
-<para>This operation has the following options:
-
-<variablelist>
-
-  <varlistentry><term><option>--recursive</option></term>
-
-    <listitem><para>
-      Use recursive instead of flat hashing mode, used when adding directories
-      to the store.
-    </para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
-/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz</screen>
-
-</refsection>
-
-</refsection>
-
-
-
-<!--######################################################################-->
-
-<refsection xml:id='refsec-nix-store-verify'><title>Operation <option>--verify</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--verify</option></arg>
-    <arg><option>--check-contents</option></arg>
-    <arg><option>--repair</option></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--verify</option> verifies the internal
-consistency of the Nix database, and the consistency between the Nix
-database and the Nix store.  Any inconsistencies encountered are
-automatically repaired.  Inconsistencies are generally the result of
-the Nix store or database being modified by non-Nix tools, or of bugs
-in Nix itself.</para>
-
-<para>This operation has the following options:
-
-<variablelist>
-
-  <varlistentry><term><option>--check-contents</option></term>
-
-    <listitem><para>Checks that the contents of every valid store path
-    has not been altered by computing a SHA-256 hash of the contents
-    and comparing it with the hash stored in the Nix database at build
-    time.  Paths that have been modified are printed out.  For large
-    stores, <option>--check-contents</option> is obviously quite
-    slow.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--repair</option></term>
-
-    <listitem><para>If any valid path is missing from the store, or
-    (if <option>--check-contents</option> is given) the contents of a
-    valid path has been modified, then try to repair the path by
-    redownloading it.  See <command>nix-store --repair-path</command>
-    for details.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--verify-path</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--verify-path</option></arg>
-    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--verify-path</option> compares the
-contents of the given store paths to their cryptographic hashes stored
-in Nix’s database.  For every changed path, it prints a warning
-message.  The exit status is 0 if no path has changed, and 1
-otherwise.</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<para>To verify the integrity of the <command>svn</command> command and all its dependencies:
-
-<screen>
-$ nix-store --verify-path $(nix-store -qR $(which svn))
-</screen>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--repair-path</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--repair-path</option></arg>
-    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--repair-path</option> attempts to
-“repair” the specified paths by redownloading them using the available
-substituters.  If no substitutes are available, then repair is not
-possible.</para>
-
-<warning><para>During repair, there is a very small time window during
-which the old path (if it exists) is moved out of the way and replaced
-with the new path.  If repair is interrupted in between, then the
-system may be left in a broken state (e.g., if the path contains a
-critical system component like the GNU C Library).</para></warning>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
-path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified!
-  expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588',
-  got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4'
-
-$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
-fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
-…
-</screen>
-
-</refsection>
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection xml:id='refsec-nix-store-dump'><title>Operation <option>--dump</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--dump</option></arg>
-    <arg choice='plain'><emphasis>path</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--dump</option> produces a NAR (Nix
-ARchive) file containing the contents of the file system tree rooted
-at <emphasis>path</emphasis>.  The archive is written to
-standard output.</para>
-
-<para>A NAR archive is like a TAR or Zip archive, but it contains only
-the information that Nix considers important.  For instance,
-timestamps are elided because all files in the Nix store have their
-timestamp set to 0 anyway.  Likewise, all permissions are left out
-except for the execute bit, because all files in the Nix store have
-444 or 555 permission.</para>
-
-<para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
-that “equal” paths always produce the same NAR archive.  For instance,
-directory entries are always sorted so that the actual on-disk order
-doesn’t influence the result.  This means that the cryptographic hash
-of a NAR dump of a path is usable as a fingerprint of the contents of
-the path.  Indeed, the hashes of store paths stored in Nix’s database
-(see <link linkend="refsec-nix-store-query"><literal>nix-store -q
---hash</literal></link>) are SHA-256 hashes of the NAR dump of each
-store path.</para>
-
-<para>NAR archives support filenames of unlimited length and 64-bit
-file sizes.  They can contain regular files, directories, and symbolic
-links, but not other types of files (such as device nodes).</para>
-
-<para>A Nix archive can be unpacked using <literal>nix-store
---restore</literal>.</para>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--restore</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--restore</option></arg>
-    <arg choice='plain'><emphasis>path</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--restore</option> unpacks a NAR archive
-to <emphasis>path</emphasis>, which must not already exist.  The
-archive is read from standard input.</para>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection xml:id='refsec-nix-store-export'><title>Operation <option>--export</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--export</option></arg>
-    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--export</option> writes a serialisation
-of the specified store paths to standard output in a format that can
-be imported into another Nix store with <command
-linkend="refsec-nix-store-import">nix-store --import</command>.  This
-is like <command linkend="refsec-nix-store-dump">nix-store
---dump</command>, except that the NAR archive produced by that command
-doesn’t contain the necessary meta-information to allow it to be
-imported into another Nix store (namely, the set of references of the
-path).</para>
-
-<para>This command does not produce a <emphasis>closure</emphasis> of
-the specified paths, so if a store path references other store paths
-that are missing in the target Nix store, the import will fail.  To
-copy a whole closure, do something like:
-
-<screen>
-$ nix-store --export $(nix-store -qR <emphasis>paths</emphasis>) > out</screen>
-
-To import the whole closure again, run:
-
-<screen>
-$ nix-store --import &lt; out</screen>
-
-</para>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection xml:id='refsec-nix-store-import'><title>Operation <option>--import</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--import</option></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--import</option> reads a serialisation of
-a set of store paths produced by <command
-linkend="refsec-nix-store-export">nix-store --export</command> from
-standard input and adds those store paths to the Nix store.  Paths
-that already exist in the Nix store are ignored.  If a path refers to
-another path that doesn’t exist in the Nix store, the import
-fails.</para>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--optimise</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--optimise</option></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--optimise</option> reduces Nix store disk
-space usage by finding identical files in the store and hard-linking
-them to each other.  It typically reduces the size of the store by
-something like 25-35%.  Only regular files and symlinks are
-hard-linked in this manner.  Files are considered identical when they
-have the same NAR archive serialisation: that is, regular files must
-have the same contents and permission (executable or non-executable),
-and symlinks must have the same contents.</para>
-
-<para>After completion, or when the command is interrupted, a report
-on the achieved savings is printed on standard error.</para>
-
-<para>Use <option>-vv</option> or <option>-vvv</option> to get some
-progress indication.</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store --optimise
-hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1'
-<emphasis>...</emphasis>
-541838819 bytes (516.74 MiB) freed by hard-linking 54143 files;
-there are 114486 files with equal contents out of 215894 files in total
-</screen>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--read-log</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <group choice='req'>
-      <arg choice='plain'><option>--read-log</option></arg>
-      <arg choice='plain'><option>-l</option></arg>
-    </group>
-    <arg choice='plain' rep='repeat'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--read-log</option> prints the build log
-of the specified store paths on standard output.  The build log is
-whatever the builder of a derivation wrote to standard output and
-standard error.  If a store path is not a derivation, the deriver of
-the store path is used.</para>
-
-<para>Build logs are kept in
-<filename>/nix/var/log/nix/drvs</filename>.  However, there is no
-guarantee that a build log is available for any particular store path.
-For instance, if the path was downloaded as a pre-built binary through
-a substitute, then the log is unavailable.</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store -l $(which ktorrent)
-building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1
-unpacking sources
-unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz
-ktorrent-2.2.1/
-ktorrent-2.2.1/NEWS
-<emphasis>...</emphasis>
-</screen>
-
-</refsection>
-
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--dump-db</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--dump-db</option></arg>
-    <arg rep='repeat'><emphasis>paths</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--dump-db</option> writes a dump of the
-Nix database to standard output.  It can be loaded into an empty Nix
-store using <option>--load-db</option>.  This is useful for making
-backups and when migrating to different database schemas.</para>
-
-<para>By default, <option>--dump-db</option> will dump the entire Nix
-database.  When one or more store paths is passed, only the subset of
-the Nix database for those store paths is dumped.  As with
-<option>--export</option>, the user is responsible for passing all the
-store paths for a closure.  See <option>--export</option> for an
-example.</para>
-
-</refsection>
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--load-db</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--load-db</option></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--load-db</option> reads a dump of the Nix
-database created by <option>--dump-db</option> from standard input and
-loads it into the Nix database.</para>
-
-</refsection>
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--print-env</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'><option>--print-env</option></arg>
-    <arg choice='plain'><emphasis>drvpath</emphasis></arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>The operation <option>--print-env</option> prints out the
-environment of a derivation in a format that can be evaluated by a
-shell.  The command line arguments of the builder are placed in the
-variable <literal>_args</literal>.</para>
-
-</refsection>
-
-<refsection><title>Example</title>
-
-<screen>
-$ nix-store --print-env $(nix-instantiate '&lt;nixpkgs>' -A firefox)
-<emphasis>…</emphasis>
-export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2'
-export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv'
-export system; system='x86_64-linux'
-export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh'
-</screen>
-
-</refsection>
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title>
-
-<refsection>
-  <title>Synopsis</title>
-  <cmdsynopsis>
-    <command>nix-store</command>
-    <arg choice='plain'>
-      <option>--generate-binary-cache-key</option>
-      <option>key-name</option>
-      <option>secret-key-file</option>
-      <option>public-key-file</option>
-    </arg>
-  </cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-
-<para>This command generates an <link
-xlink:href="http://ed25519.cr.yp.to/">Ed25519 key pair</link> that can
-be used to create a signed binary cache. It takes three mandatory
-parameters:
-
-<orderedlist>
-
-  <listitem><para>A key name, such as
-  <literal>cache.example.org-1</literal>, that is used to look up keys
-  on the client when it verifies signatures. It can be anything, but
-  it’s suggested to use the host name of your cache
-  (e.g. <literal>cache.example.org</literal>) with a suffix denoting
-  the number of the key (to be incremented every time you need to
-  revoke a key).</para></listitem>
-
-  <listitem><para>The file name where the secret key is to be
-  stored.</para></listitem>
-
-  <listitem><para>The file name where the public key is to be
-  stored.</para></listitem>
-
-</orderedlist>
-
-</para>
-
-</refsection>
-
-</refsection>
-
-
-<!--######################################################################-->
-
-<refsection condition="manpage"><title>Environment variables</title>
-
-<variablelist>
-  <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
-</variablelist>
-
-</refsection>
-
-
-</refentry>
diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml
deleted file mode 100644
index 6b7b5c833..000000000
--- a/doc/manual/command-ref/opt-common-syn.xml
+++ /dev/null
@@ -1,68 +0,0 @@
-<nop xmlns="http://docbook.org/ns/docbook">
-
-<arg><option>--help</option></arg>
-<arg><option>--version</option></arg>
-<arg rep='repeat'>
-  <group choice='req'>
-    <arg choice='plain'><option>--verbose</option></arg>
-    <arg choice='plain'><option>-v</option></arg>
-  </group>
-</arg>
-<arg>
-  <arg choice='plain'><option>--quiet</option></arg>
-</arg>
-<arg>
-  <option>--log-format</option>
-  <emphasis>format</emphasis>
-</arg>
-<arg>
-  <group choice='plain'>
-    <arg choice='plain'><option>--no-build-output</option></arg>
-    <arg choice='plain'><option>-Q</option></arg>
-  </group>
-</arg>
-<arg>
-  <group choice='req'>
-    <arg choice='plain'><option>--max-jobs</option></arg>
-    <arg choice='plain'><option>-j</option></arg>
-  </group>
-  <emphasis>number</emphasis>
-</arg>
-<arg>
-  <option>--cores</option>
-  <emphasis>number</emphasis>
-</arg>
-<arg>
-  <option>--max-silent-time</option>
-  <emphasis>number</emphasis>
-</arg>
-<arg>
-  <option>--timeout</option>
-  <emphasis>number</emphasis>
-</arg>
-<arg>
-  <group choice='plain'>
-    <arg choice='plain'><option>--keep-going</option></arg>
-    <arg choice='plain'><option>-k</option></arg>
-  </group>
-</arg>
-<arg>
-  <group choice='plain'>
-    <arg choice='plain'><option>--keep-failed</option></arg>
-    <arg choice='plain'><option>-K</option></arg>
-  </group>
-</arg>
-<arg><option>--fallback</option></arg>
-<arg><option>--readonly-mode</option></arg>
-<arg>
-  <option>-I</option>
-  <emphasis>path</emphasis>
-</arg>
-<arg>
-  <option>--option</option>
-  <emphasis>name</emphasis>
-  <emphasis>value</emphasis>
-</arg>
-<sbr />
-
-</nop>
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
deleted file mode 100644
index a095039b9..000000000
--- a/doc/manual/command-ref/opt-common.xml
+++ /dev/null
@@ -1,405 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook" xml:id="sec-common-options">
-
-<title>Common Options</title>
-
-
-<para>Most Nix commands accept the following command-line options:</para>
-
-<variablelist xml:id="opt-common">
-
-<varlistentry><term><option>--help</option></term>
-
-  <listitem><para>Prints out a summary of the command syntax and
-  exits.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--version</option></term>
-
-  <listitem><para>Prints out the Nix version number on standard output
-  and exits.</para></listitem>
-</varlistentry>
-
-
-<varlistentry><term><option>--verbose</option> / <option>-v</option></term>
-
-  <listitem>
-
-  <para>Increases the level of verbosity of diagnostic messages
-  printed on standard error.  For each Nix operation, the information
-  printed on standard output is well-defined; any diagnostic
-  information is printed on standard error, never on standard
-  output.</para>
-
-  <para>This option may be specified repeatedly.  Currently, the
-  following verbosity levels exist:</para>
-
-  <variablelist>
-
-    <varlistentry><term>0</term>
-    <listitem><para>“Errors only”: only print messages
-    explaining why the Nix invocation failed.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>1</term>
-    <listitem><para>“Informational”: print
-    <emphasis>useful</emphasis> messages about what Nix is doing.
-    This is the default.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>2</term>
-    <listitem><para>“Talkative”: print more informational
-    messages.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>3</term>
-    <listitem><para>“Chatty”: print even more
-    informational messages.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>4</term>
-    <listitem><para>“Debug”: print debug
-    information.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>5</term>
-    <listitem><para>“Vomit”: print vast amounts of debug
-    information.</para></listitem>
-    </varlistentry>
-
-  </variablelist>
-
-  </listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--quiet</option></term>
-
-  <listitem>
-
-  <para>Decreases the level of verbosity of diagnostic messages
-  printed on standard error.  This is the inverse option to
-  <option>-v</option> / <option>--verbose</option>.
-  </para>
-
-  <para>This option may be specified repeatedly.  See the previous
-  verbosity levels list.</para>
-
-  </listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <emphasis>format</emphasis></term>
-
-  <listitem>
-
-  <para>This option can be used to change the output of the log format, with
-  <emphasis>format</emphasis> being one of:</para>
-
-  <variablelist>
-
-    <varlistentry><term>raw</term>
-    <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>internal-json</term>
-    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>bar</term>
-    <listitem><para>Only display a progress bar during the builds.</para></listitem>
-    </varlistentry>
-
-    <varlistentry><term>bar-with-logs</term>
-    <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
-    </varlistentry>
-
-  </variablelist>
-
-  </listitem>
-
-</varlistentry>
-
-<varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
-
-  <listitem><para>By default, output written by builders to standard
-  output and standard error is echoed to the Nix command's standard
-  error.  This option suppresses this behaviour.  Note that the
-  builder's standard output and error are always written to a log file
-  in
-  <filename><emphasis>prefix</emphasis>/nix/var/log/nix</filename>.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
-<emphasis>number</emphasis></term>
-
-  <listitem>
-
-  <para>Sets the maximum number of build jobs that Nix will
-  perform in parallel to the specified number.  Specify
-  <literal>auto</literal> to use the number of CPUs in the system.
-  The default is specified by the <link
-  linkend='conf-max-jobs'><literal>max-jobs</literal></link>
-  configuration setting, which itself defaults to
-  <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.</para>
-
-  <para> Setting it to <literal>0</literal> disallows building on the local
-  machine, which is useful when you want builds to happen only on remote
-  builders.</para>
-
-  </listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="opt-cores"><term><option>--cores</option></term>
-
-  <listitem><para>Sets the value of the <literal>NIX_BUILD_CORES</literal>
-  environment variable in the invocation of builders.  Builders can
-  use this variable at their discretion to control the maximum amount
-  of parallelism.  For instance, in Nixpkgs, if the derivation
-  attribute <varname>enableParallelBuilding</varname> is set to
-  <literal>true</literal>, the builder passes the
-  <option>-j<emphasis>N</emphasis></option> flag to GNU Make.
-  It defaults to the value of the <link
-  linkend='conf-cores'><literal>cores</literal></link>
-  configuration setting, if set, or <literal>1</literal> otherwise.
-  The value <literal>0</literal> means that the builder should use all
-  available CPU cores in the system.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="opt-max-silent-time"><term><option>--max-silent-time</option></term>
-
-  <listitem><para>Sets the maximum number of seconds that a builder
-  can go without producing any data on standard output or standard
-  error.  The default is specified by the <link
-  linkend='conf-max-silent-time'><literal>max-silent-time</literal></link>
-  configuration setting.  <literal>0</literal> means no
-  time-out.</para></listitem>
-
-</varlistentry>
-
-<varlistentry xml:id="opt-timeout"><term><option>--timeout</option></term>
-
-  <listitem><para>Sets the maximum number of seconds that a builder
-  can run.  The default is specified by the <link
-  linkend='conf-timeout'><literal>timeout</literal></link>
-  configuration setting.  <literal>0</literal> means no
-  timeout.</para></listitem>
-
-</varlistentry>
-
-<varlistentry><term><option>--keep-going</option> / <option>-k</option></term>
-
-  <listitem><para>Keep going in case of failed builds, to the
-  greatest extent possible.  That is, if building an input of some
-  derivation fails, Nix will still build the other inputs, but not the
-  derivation itself.  Without this option, Nix stops if any build
-  fails (except for builds of substitutes), possibly killing builds in
-  progress (in case of parallel or distributed builds).</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--keep-failed</option> / <option>-K</option></term>
-
-  <listitem><para>Specifies that in case of a build failure, the
-  temporary directory (usually in <filename>/tmp</filename>) in which
-  the build takes place should not be deleted.  The path of the build
-  directory is printed as an informational message.
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry><term><option>--fallback</option></term>
-
-  <listitem>
-
-  <para>Whenever Nix attempts to build a derivation for which
-  substitutes are known for each output path, but realising the output
-  paths through the substitutes fails, fall back on building the
-  derivation.</para>
-
-  <para>The most common scenario in which this is useful is when we
-  have registered substitutes in order to perform binary distribution
-  from, say, a network repository.  If the repository is down, the
-  realisation of the derivation will fail.  When this option is
-  specified, Nix will build the derivation instead.  Thus,
-  installation from binaries falls back on installation from source.
-  This option is not the default since it is generally not desirable
-  for a transient failure in obtaining the substitutes to lead to a
-  full build from source (with the related consumption of
-  resources).</para>
-
-  </listitem>
-
-</varlistentry>
-
-<varlistentry><term><option>--no-build-hook</option></term>
-
-  <listitem>
-
-  <para>Disables the build hook mechanism.  This allows to ignore remote
-  builders if they are setup on the machine.</para>
-
-  <para>It's useful in cases where the bandwidth between the client and the
-  remote builder is too low.  In that case it can take more time to upload the
-  sources to the remote builder and fetch back the result than to do the
-  computation locally.</para>
-
-  </listitem>
-
-</varlistentry>
-
-
-
-<varlistentry><term><option>--readonly-mode</option></term>
-
-  <listitem><para>When this option is used, no attempt is made to open
-  the Nix database.  Most Nix operations do need database access, so
-  those operations will fail.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--arg</option> <emphasis>name</emphasis> <emphasis>value</emphasis></term>
-
-  <listitem><para>This option is accepted by
-  <command>nix-env</command>, <command>nix-instantiate</command>,
-  <command>nix-shell</command> and <command>nix-build</command>.
-  When evaluating Nix expressions, the expression evaluator will
-  automatically try to call functions that
-  it encounters.  It can automatically call functions for which every
-  argument has a <link linkend='ss-functions'>default value</link>
-  (e.g., <literal>{ <emphasis>argName</emphasis> ?
-  <emphasis>defaultValue</emphasis> }:
-  <emphasis>...</emphasis></literal>).  With
-  <option>--arg</option>, you can also call functions that have
-  arguments without a default value (or override a default value).
-  That is, if the evaluator encounters a function with an argument
-  named <emphasis>name</emphasis>, it will call it with value
-  <emphasis>value</emphasis>.</para>
-
-  <para>For instance, the top-level <literal>default.nix</literal> in
-  Nixpkgs is actually a function:
-
-<programlisting>
-{ # The system (e.g., `i686-linux') for which to build the packages.
-  system ? builtins.currentSystem
-  <emphasis>...</emphasis>
-}: <emphasis>...</emphasis></programlisting>
-
-  So if you call this Nix expression (e.g., when you do
-  <literal>nix-env -i <emphasis>pkgname</emphasis></literal>),
-  the function will be called automatically using the value <link
-  linkend='builtin-currentSystem'><literal>builtins.currentSystem</literal></link>
-  for the <literal>system</literal> argument.  You can override this
-  using <option>--arg</option>, e.g., <literal>nix-env -i
-  <emphasis>pkgname</emphasis> --arg system
-  \"i686-freebsd\"</literal>.  (Note that since the argument is a Nix
-  string literal, you have to escape the quotes.)</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--argstr</option> <emphasis>name</emphasis> <emphasis>value</emphasis></term>
-
-  <listitem><para>This option is like <option>--arg</option>, only the
-  value is not a Nix expression but a string.  So instead of
-  <literal>--arg system \"i686-linux\"</literal> (the outer quotes are
-  to keep the shell happy) you can say <literal>--argstr system
-  i686-linux</literal>.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="opt-attr"><term><option>--attr</option> / <option>-A</option>
-<emphasis>attrPath</emphasis></term>
-
-  <listitem><para>Select an attribute from the top-level Nix
-  expression being evaluated.  (<command>nix-env</command>,
-  <command>nix-instantiate</command>, <command>nix-build</command> and
-  <command>nix-shell</command> only.)  The <emphasis>attribute
-  path</emphasis> <emphasis>attrPath</emphasis> is a sequence of
-  attribute names separated by dots.  For instance, given a top-level
-  Nix expression <emphasis>e</emphasis>, the attribute path
-  <literal>xorg.xorgserver</literal> would cause the expression
-  <literal><emphasis>e</emphasis>.xorg.xorgserver</literal> to
-  be used.  See <link
-  linkend='refsec-nix-env-install-examples'><command>nix-env
-  --install</command></link> for some concrete examples.</para>
-
-  <para>In addition to attribute names, you can also specify array
-  indices.  For instance, the attribute path
-  <literal>foo.3.bar</literal> selects the <literal>bar</literal>
-  attribute of the fourth element of the array in the
-  <literal>foo</literal> attribute of the top-level
-  expression.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--expr</option> / <option>-E</option></term>
-
-  <listitem><para>Interpret the command line arguments as a list of
-  Nix expressions to be parsed and evaluated, rather than as a list
-  of file names of Nix expressions.
-  (<command>nix-instantiate</command>, <command>nix-build</command>
-  and <command>nix-shell</command> only.)</para>
-
-  <para>For <command>nix-shell</command>, this option is commonly used
-  to give you a shell in which you can build the packages returned
-  by the expression. If you want to get a shell which contain the
-  <emphasis>built</emphasis> packages ready for use, give your
-  expression to the <command>nix-shell -p</command> convenience flag
-  instead.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry xml:id="opt-I"><term><option>-I</option> <emphasis>path</emphasis></term>
-
-  <listitem><para>Add a path to the Nix expression search path.  This
-  option may be given multiple times.  See the <literal
-  linkend="env-NIX_PATH">NIX_PATH</literal> environment variable for
-  information on the semantics of the Nix search path.  Paths added
-  through <option>-I</option> take precedence over
-  <literal>NIX_PATH</literal>.</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--option</option> <emphasis>name</emphasis> <emphasis>value</emphasis></term>
-
-  <listitem><para>Set the Nix configuration option
-  <emphasis>name</emphasis> to <emphasis>value</emphasis>.
-  This overrides settings in the Nix configuration file (see
-  <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>).</para></listitem>
-
-</varlistentry>
-
-
-<varlistentry><term><option>--repair</option></term>
-
-  <listitem><para>Fix corrupted or missing store paths by
-  redownloading or rebuilding them.  Note that this is slow because it
-  requires computing a cryptographic hash of the contents of every
-  path in the closure of the build.  Also note the warning under
-  <command>nix-store --repair-path</command>.</para></listitem>
-
-</varlistentry>
-
-
-</variablelist>
-
-
-</chapter>
diff --git a/doc/manual/command-ref/opt-inst-syn.xml b/doc/manual/command-ref/opt-inst-syn.xml
deleted file mode 100644
index e324a5e6d..000000000
--- a/doc/manual/command-ref/opt-inst-syn.xml
+++ /dev/null
@@ -1,22 +0,0 @@
-<nop xmlns="http://docbook.org/ns/docbook">
-  
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--prebuilt-only</option></arg>
-      <arg choice='plain'><option>-b</option></arg>
-    </group>
-  </arg>
-  
-  <arg>
-    <group choice='req'>
-      <arg choice='plain'><option>--attr</option></arg>
-      <arg choice='plain'><option>-A</option></arg>
-    </group>
-  </arg>
-
-  <arg><option>--from-expression</option></arg>
-  <arg><option>-E</option></arg>
-    
-  <arg><option>--from-profile</option> <emphasis>path</emphasis></arg>
-
-</nop>
diff --git a/doc/manual/command-ref/utilities.xml b/doc/manual/command-ref/utilities.xml
deleted file mode 100644
index 893f5b5b5..000000000
--- a/doc/manual/command-ref/utilities.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='ch-utilities'>
-
-<title>Utilities</title>
-
-<para>This section lists utilities that you can use when you
-work with Nix.</para>
-
-<xi:include href="nix-channel.xml" />
-<xi:include href="nix-collect-garbage.xml" />
-<xi:include href="nix-copy-closure.xml" />
-<xi:include href="nix-daemon.xml" />
-<xi:include href="nix-hash.xml" />
-<xi:include href="nix-instantiate.xml" />
-<xi:include href="nix-prefetch-url.xml" />
-
-</chapter>
diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
deleted file mode 100644
index 794d020d9..000000000
--- a/doc/manual/expressions/advanced-attributes.xml
+++ /dev/null
@@ -1,351 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-advanced-attributes">
-
-<title>Advanced Attributes</title>
-
-<para>Derivations can declare some infrequently used optional
-attributes.</para>
-
-<variablelist>
-
-  <varlistentry xml:id="adv-attr-allowedReferences"><term><varname>allowedReferences</varname></term>
-
-    <listitem><para>The optional attribute
-    <varname>allowedReferences</varname> specifies a list of legal
-    references (dependencies) of the output of the builder.  For
-    example,
-
-<programlisting>
-allowedReferences = [];
-</programlisting>
-
-    enforces that the output of a derivation cannot have any runtime
-    dependencies on its inputs.  To allow an output to have a runtime
-    dependency on itself, use <literal>"out"</literal> as a list item.
-    This is used in NixOS to check that generated files such as
-    initial ramdisks for booting Linux don’t have accidental
-    dependencies on other paths in the Nix store.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-allowedRequisites"><term><varname>allowedRequisites</varname></term>
-
-    <listitem><para>This attribute is similar to
-    <varname>allowedReferences</varname>, but it specifies the legal
-    requisites of the whole closure, so all the dependencies
-    recursively.  For example,
-
-<programlisting>
-allowedRequisites = [ foobar ];
-</programlisting>
-
-    enforces that the output of a derivation cannot have any other
-    runtime dependency than <varname>foobar</varname>, and in addition
-    it enforces that <varname>foobar</varname> itself doesn't
-    introduce any other dependency itself.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="adv-attr-disallowedReferences"><term><varname>disallowedReferences</varname></term>
-
-    <listitem><para>The optional attribute
-    <varname>disallowedReferences</varname> specifies a list of illegal
-    references (dependencies) of the output of the builder.  For
-    example,
-
-<programlisting>
-disallowedReferences = [ foo ];
-</programlisting>
-
-    enforces that the output of a derivation cannot have a direct runtime
-    dependencies on the derivation <varname>foo</varname>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-disallowedRequisites"><term><varname>disallowedRequisites</varname></term>
-
-    <listitem><para>This attribute is similar to
-    <varname>disallowedReferences</varname>, but it specifies illegal
-    requisites for the whole closure, so all the dependencies
-    recursively.  For example,
-
-<programlisting>
-disallowedRequisites = [ foobar ];
-</programlisting>
-
-    enforces that the output of a derivation cannot have any
-    runtime dependency on <varname>foobar</varname> or any other derivation
-    depending recursively on <varname>foobar</varname>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-exportReferencesGraph"><term><varname>exportReferencesGraph</varname></term>
-
-    <listitem><para>This attribute allows builders access to the
-    references graph of their inputs.  The attribute is a list of
-    inputs in the Nix store whose references graph the builder needs
-    to know.  The value of this attribute should be a list of pairs
-    <literal>[ <emphasis>name1</emphasis>
-    <emphasis>path1</emphasis> <emphasis>name2</emphasis>
-    <emphasis>path2</emphasis> <emphasis>...</emphasis>
-    ]</literal>.  The references graph of each
-    <emphasis>pathN</emphasis> will be stored in a text file
-    <emphasis>nameN</emphasis> in the temporary build directory.
-    The text files have the format used by <command>nix-store
-    --register-validity</command> (with the deriver fields left
-    empty).  For example, when the following derivation is built:
-
-<programlisting>
-derivation {
-  ...
-  exportReferencesGraph = [ "libfoo-graph" libfoo ];
-};
-</programlisting>
-
-    the references graph of <literal>libfoo</literal> is placed in the
-    file <filename>libfoo-graph</filename> in the temporary build
-    directory.</para>
-
-    <para><varname>exportReferencesGraph</varname> is useful for
-    builders that want to do something with the closure of a store
-    path.  Examples include the builders in NixOS that generate the
-    initial ramdisk for booting Linux (a <command>cpio</command>
-    archive containing the closure of the boot script) and the
-    ISO-9660 image for the installation CD (which is populated with a
-    Nix store containing the closure of a bootable NixOS
-    configuration).</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-impureEnvVars"><term><varname>impureEnvVars</varname></term>
-
-    <listitem><para>This attribute allows you to specify a list of
-    environment variables that should be passed from the environment
-    of the calling user to the builder.  Usually, the environment is
-    cleared completely when the builder is executed, but with this
-    attribute you can allow specific environment variables to be
-    passed unmodified.  For example, <function>fetchurl</function> in
-    Nixpkgs has the line
-
-<programlisting>
-impureEnvVars = [ "http_proxy" "https_proxy" <emphasis>...</emphasis> ];
-</programlisting>
-
-    to make it use the proxy server configuration specified by the
-    user in the environment variables <literal>http_proxy</literal> and
-    friends.</para>
-
-    <para>This attribute is only allowed in <link
-    linkend="fixed-output-drvs">fixed-output derivations</link>, where
-    impurities such as these are okay since (the hash of) the output
-    is known in advance.  It is ignored for all other
-    derivations.</para>
-
-    <warning><para><varname>impureEnvVars</varname> implementation takes
-    environment variables from the current builder process. When a daemon is
-    building its environmental variables are used. Without the daemon, the
-    environmental variables come from the environment of the
-    <command>nix-build</command>.</para></warning></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="fixed-output-drvs">
-    <term xml:id="adv-attr-outputHash"><varname>outputHash</varname></term>
-    <term xml:id="adv-attr-outputHashAlgo"><varname>outputHashAlgo</varname></term>
-    <term xml:id="adv-attr-outputHashMode"><varname>outputHashMode</varname></term>
-
-    <listitem><para>These attributes declare that the derivation is a
-    so-called <emphasis>fixed-output derivation</emphasis>, which
-    means that a cryptographic hash of the output is already known in
-    advance.  When the build of a fixed-output derivation finishes,
-    Nix computes the cryptographic hash of the output and compares it
-    to the hash declared with these attributes.  If there is a
-    mismatch, the build fails.</para>
-
-    <para>The rationale for fixed-output derivations is derivations
-    such as those produced by the <function>fetchurl</function>
-    function.  This function downloads a file from a given URL.  To
-    ensure that the downloaded file has not been modified, the caller
-    must also specify a cryptographic hash of the file.  For example,
-
-<programlisting>
-fetchurl {
-  url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
-  sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-}
-</programlisting>
-
-    It sometimes happens that the URL of the file changes, e.g.,
-    because servers are reorganised or no longer available.  We then
-    must update the call to <function>fetchurl</function>, e.g.,
-
-<programlisting>
-fetchurl {
-  url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-  sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-}
-</programlisting>
-
-    If a <function>fetchurl</function> derivation was treated like a
-    normal derivation, the output paths of the derivation and
-    <emphasis>all derivations depending on it</emphasis> would change.
-    For instance, if we were to change the URL of the Glibc source
-    distribution in Nixpkgs (a package on which almost all other
-    packages depend) massive rebuilds would be needed.  This is
-    unfortunate for a change which we know cannot have a real effect
-    as it propagates upwards through the dependency graph.</para>
-
-    <para>For fixed-output derivations, on the other hand, the name of
-    the output path only depends on the <varname>outputHash*</varname>
-    and <varname>name</varname> attributes, while all other attributes
-    are ignored for the purpose of computing the output path.  (The
-    <varname>name</varname> attribute is included because it is part
-    of the path.)</para>
-
-    <para>As an example, here is the (simplified) Nix expression for
-    <varname>fetchurl</varname>:
-
-<programlisting>
-{ stdenv, curl }: # The <command>curl</command> program is used for downloading.
-
-{ url, sha256 }:
-
-stdenv.mkDerivation {
-  name = baseNameOf (toString url);
-  builder = ./builder.sh;
-  buildInputs = [ curl ];
-
-  # This is a fixed-output derivation; the output must be a regular
-  # file with SHA256 hash <varname>sha256</varname>.
-  outputHashMode = "flat";
-  outputHashAlgo = "sha256";
-  outputHash = sha256;
-
-  inherit url;
-}
-</programlisting>
-
-    </para>
-
-    <para>The <varname>outputHashAlgo</varname> attribute specifies
-    the hash algorithm used to compute the hash.  It can currently be
-    <literal>"sha1"</literal>, <literal>"sha256"</literal> or
-    <literal>"sha512"</literal>.</para>
-
-    <para>The <varname>outputHashMode</varname> attribute determines
-    how the hash is computed.  It must be one of the following two
-    values:
-
-    <variablelist>
-
-      <varlistentry><term><literal>"flat"</literal></term>
-
-        <listitem><para>The output must be a non-executable regular
-        file.  If it isn’t, the build fails.  The hash is simply
-        computed over the contents of that file (so it’s equal to what
-        Unix commands like <command>sha256sum</command> or
-        <command>sha1sum</command> produce).</para>
-
-        <para>This is the default.</para></listitem>
-
-      </varlistentry>
-
-      <varlistentry><term><literal>"recursive"</literal></term>
-
-        <listitem><para>The hash is computed over the NAR archive dump
-        of the output (i.e., the result of <link
-        linkend="refsec-nix-store-dump"><command>nix-store
-        --dump</command></link>).  In this case, the output can be
-        anything, including a directory tree.</para></listitem>
-
-      </varlistentry>
-
-    </variablelist>
-
-    </para>
-
-    <para>The <varname>outputHash</varname> attribute, finally, must
-    be a string containing the hash in either hexadecimal or base-32
-    notation.  (See the <link
-    linkend="sec-nix-hash"><command>nix-hash</command> command</link>
-    for information about converting to and from base-32
-    notation.)</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-passAsFile"><term><varname>passAsFile</varname></term>
-
-    <listitem><para>A list of names of attributes that should be
-    passed via files rather than environment variables.  For example,
-    if you have
-
-    <programlisting>
-passAsFile = ["big"];
-big = "a very long string";
-    </programlisting>
-
-    then when the builder runs, the environment variable
-    <literal>bigPath</literal> will contain the absolute path to a
-    temporary file containing <literal>a very long
-    string</literal>. That is, for any attribute
-    <emphasis>x</emphasis> listed in
-    <varname>passAsFile</varname>, Nix will pass an environment
-    variable <literal><emphasis>x</emphasis>Path</literal> holding
-    the path of the file containing the value of attribute
-    <emphasis>x</emphasis>. This is useful when you need to pass
-    large strings to a builder, since most operating systems impose a
-    limit on the size of the environment (typically, a few hundred
-    kilobyte).</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-preferLocalBuild"><term><varname>preferLocalBuild</varname></term>
-
-    <listitem><para>If this attribute is set to
-    <literal>true</literal> and <link
-    linkend="chap-distributed-builds">distributed building is
-    enabled</link>, then, if possible, the derivaton will be built
-    locally instead of forwarded to a remote machine.  This is
-    appropriate for trivial builders where the cost of doing a
-    download or remote build would exceed the cost of building
-    locally.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="adv-attr-allowSubstitutes"><term><varname>allowSubstitutes</varname></term>
-
-    <listitem>
-    <para>If this attribute is set to
-    <literal>false</literal>, then Nix will always build this
-    derivation; it will not try to substitute its outputs. This is
-    useful for very trivial derivations (such as
-    <function>writeText</function> in Nixpkgs) that are cheaper to
-    build than to substitute from a binary cache.</para>
-
-    <note><para>You need to have a builder configured which satisfies
-    the derivation’s <literal>system</literal> attribute, since the
-    derivation cannot be substituted. Thus it is usually a good idea
-    to align <literal>system</literal> with
-    <literal>builtins.currentSystem</literal> when setting
-    <literal>allowSubstitutes</literal> to <literal>false</literal>.
-    For most trivial derivations this should be the case.
-    </para></note>
-    </listitem>
-
-  </varlistentry>
-
-
-</variablelist>
-
-</section>
diff --git a/doc/manual/expressions/arguments-variables.xml b/doc/manual/expressions/arguments-variables.xml
deleted file mode 100644
index a4375c777..000000000
--- a/doc/manual/expressions/arguments-variables.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='sec-arguments'>
-
-<title>Arguments and Variables</title>
-
-<para>The Nix expression in <xref linkend='ex-hello-nix' /> is a
-function; it is missing some arguments that have to be filled in
-somewhere.  In the Nix Packages collection this is done in the file
-<filename>pkgs/top-level/all-packages.nix</filename>, where all Nix
-expressions for packages are imported and called with the appropriate
-arguments. Here are some fragments of
-<filename>all-packages.nix</filename>, with annotations of what they
-mean:</para>
-
-<programlisting>
-...
-
-rec { ①
-
-  hello = import ../applications/misc/hello/ex-1 ② { ③
-    inherit fetchurl stdenv perl;
-  };
-
-  perl = import ../development/interpreters/perl { ④
-    inherit fetchurl stdenv;
-  };
-
-  fetchurl = import ../build-support/fetchurl {
-    inherit stdenv; ...
-  };
-
-  stdenv = ...;
-
-}
-</programlisting>
-
-<orderedlist>
-
-  <listitem>
-    <para>This file defines a set of attributes, all of which are
-    concrete derivations (i.e., not functions).  In fact, we define a
-    <emphasis>mutually recursive</emphasis> set of attributes.  That
-    is, the attributes can refer to each other.  This is precisely
-    what we want since we want to <quote>plug</quote> the
-    various packages into each other.</para>
-  </listitem>
-
-  <listitem>
-
-    <para>Here we <emphasis>import</emphasis> the Nix expression for
-    GNU Hello.  The import operation just loads and returns the
-    specified Nix expression. In fact, we could just have put the
-    contents of <xref linkend='ex-hello-nix' /> in
-    <filename>all-packages.nix</filename> at this point.  That
-    would be completely equivalent, but it would make the file rather
-    bulky.</para>
-
-    <para>Note that we refer to
-    <filename>../applications/misc/hello/ex-1</filename>, not
-    <filename>../applications/misc/hello/ex-1/default.nix</filename>.
-    When you try to import a directory, Nix automatically appends
-    <filename>/default.nix</filename> to the file name.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>This is where the actual composition takes place.  Here we
-    <emphasis>call</emphasis> the function imported from
-    <filename>../applications/misc/hello/ex-1</filename> with a set
-    containing the things that the function expects, namely
-    <varname>fetchurl</varname>, <varname>stdenv</varname>, and
-    <varname>perl</varname>.  We use inherit again to use the
-    attributes defined in the surrounding scope (we could also have
-    written <literal>fetchurl = fetchurl;</literal>, etc.).</para>
-
-    <para>The result of this function call is an actual derivation
-    that can be built by Nix (since when we fill in the arguments of
-    the function, what we get is its body, which is the call to
-    <varname>stdenv.mkDerivation</varname> in <xref
-    linkend='ex-hello-nix' />).</para>
-
-    <note><para>Nixpkgs has a convenience function
-    <function>callPackage</function> that imports and calls a
-    function, filling in any missing arguments by passing the
-    corresponding attribute from the Nixpkgs set, like this:
-
-<programlisting>
-hello = callPackage ../applications/misc/hello/ex-1 { };
-</programlisting>
-
-    If necessary, you can set or override arguments:
-
-<programlisting>
-hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; };
-</programlisting>
-
-    </para></note>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Likewise, we have to instantiate Perl,
-    <varname>fetchurl</varname>, and the standard environment.</para>
-
-  </listitem>
-
-</orderedlist>
-
-</section>
diff --git a/doc/manual/expressions/build-script.xml b/doc/manual/expressions/build-script.xml
deleted file mode 100644
index 5c55954fc..000000000
--- a/doc/manual/expressions/build-script.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='sec-build-script'>
-
-<title>Build Script</title>
-
-<para>Here is the builder referenced
-from Hello's Nix expression (stored in
-<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>):</para>
-
-<programlisting>
-source $stdenv/setup ①
-
-PATH=$perl/bin:$PATH ②
-
-tar xvfz $src ③
-cd hello-*
-./configure --prefix=$out ④
-make ⑤
-make install</programlisting>
-
-<para>The builder can actually be made a lot shorter by using the
-<emphasis>generic builder</emphasis> functions provided by
-<varname>stdenv</varname>, but here we write out the build steps to
-elucidate what a builder does.  It performs the following
-steps:</para>
-
-<orderedlist>
-
-  <listitem>
-
-    <para>When Nix runs a builder, it initially completely clears the
-    environment (except for the attributes declared in the
-    derivation).  This is done to prevent undeclared inputs from being
-    used in the build process. If for example the
-    <literal>PATH</literal> contained <filename>/usr/bin</filename>,
-    then you might accidentally use
-    <filename>/usr/bin/gcc</filename>.</para>
-
-    <para>So the first step is to set up the environment.  This is
-    done by calling the <filename>setup</filename> script of the
-    standard environment.  The environment variable
-    <literal>stdenv</literal> points to the location of the standard
-    environment being used.  (It wasn't specified explicitly as an
-    attribute in <xref linkend='ex-hello-nix' />, but
-    <varname>mkDerivation</varname> adds it automatically.)</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Since Hello needs Perl, we have to make sure that Perl is in
-    the <literal>PATH</literal>.  The <literal>perl</literal> environment
-    variable points to the location of the Perl package (since it
-    was passed in as an attribute to the derivation), so
-    <filename><emphasis>$perl</emphasis>/bin</filename> is the
-    directory containing the Perl interpreter.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Now we have to unpack the sources.  The
-    <varname>src</varname> attribute was bound to the result of
-    fetching the Hello source tarball from the network, so the
-    <literal>src</literal> environment variable points to the location in
-    the Nix store to which the tarball was downloaded.  After
-    unpacking, we <command>cd</command> to the resulting source
-    directory.</para>
-
-    <para>The whole build is performed in a temporary directory
-    created in <varname>/tmp</varname>, by the way.  This directory is
-    removed after the builder finishes, so there is no need to clean
-    up the sources afterwards.  Also, the temporary directory is
-    always newly created, so you don't have to worry about files from
-    previous builds interfering with the current build.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>GNU Hello is a typical Autoconf-based package, so we first
-    have to run its <filename>configure</filename> script.  In Nix
-    every package is stored in a separate location in the Nix store,
-    for instance
-    <filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>.
-    Nix computes this path by cryptographically hashing all attributes
-    of the derivation.  The path is passed to the builder through the
-    <literal>out</literal> environment variable.  So here we give
-    <filename>configure</filename> the parameter
-    <literal>--prefix=$out</literal> to cause Hello to be installed in
-    the expected location.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Finally we build Hello (<literal>make</literal>) and install
-    it into the location specified by <literal>out</literal>
-    (<literal>make install</literal>).</para>
-
-  </listitem>
-
-</orderedlist>
-
-<para>If you are wondering about the absence of error checking on the
-result of various commands called in the builder: this is because the
-shell script is evaluated with Bash's <option>-e</option> option,
-which causes the script to be aborted if any command fails without an
-error check.</para>
-
-</section>
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
deleted file mode 100644
index 4ad481ad3..000000000
--- a/doc/manual/expressions/builtins.xml
+++ /dev/null
@@ -1,1663 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='ssec-builtins'>
-
-<title>Built-in Functions</title>
-
-<para>This section lists the functions and constants built into the
-Nix expression evaluator.  (The built-in function
-<function>derivation</function> is discussed above.)  Some built-ins,
-such as <function>derivation</function>, are always in scope of every
-Nix expression; you can just access them right away.  But to prevent
-polluting the namespace too much, most built-ins are not in scope.
-Instead, you can access them through the <varname>builtins</varname>
-built-in value, which is a set that contains all built-in functions
-and values.  For instance, <function>derivation</function> is also
-available as <function>builtins.derivation</function>.</para>
-
-
-<variablelist>
-
-
-  <varlistentry xml:id='builtin-abort'>
-    <term><function>abort</function> <emphasis>s</emphasis></term>
-    <term><function>builtins.abort</function> <emphasis>s</emphasis></term>
-
-    <listitem><para>Abort Nix expression evaluation, print error
-    message <emphasis>s</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-add'>
-    <term><function>builtins.add</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis>
-    </term>
-
-    <listitem><para>Return the sum of the numbers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-all'>
-    <term><function>builtins.all</function>
-    <emphasis>pred</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if the function
-    <emphasis>pred</emphasis> returns <literal>true</literal>
-    for all elements of <emphasis>list</emphasis>,
-    and <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-any'>
-    <term><function>builtins.any</function>
-    <emphasis>pred</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if the function
-    <emphasis>pred</emphasis> returns <literal>true</literal>
-    for at least one element of <emphasis>list</emphasis>,
-    and <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-attrNames'>
-    <term><function>builtins.attrNames</function>
-    <emphasis>set</emphasis></term>
-
-    <listitem><para>Return the names of the attributes in the set
-    <emphasis>set</emphasis> in an alphabetically sorted list.  For instance,
-    <literal>builtins.attrNames { y = 1; x = "foo"; }</literal>
-    evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-attrValues'>
-    <term><function>builtins.attrValues</function>
-    <emphasis>set</emphasis></term>
-
-    <listitem><para>Return the values of the attributes in the set
-    <emphasis>set</emphasis> in the order corresponding to the
-    sorted attribute names.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-baseNameOf'>
-    <term><function>baseNameOf</function> <emphasis>s</emphasis></term>
-
-    <listitem><para>Return the <emphasis>base name</emphasis> of the
-    string <emphasis>s</emphasis>, that is, everything following
-    the final slash in the string.  This is similar to the GNU
-    <command>basename</command> command.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-bitAnd'>
-    <term><function>builtins.bitAnd</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return the bitwise AND of the integers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-bitOr'>
-    <term><function>builtins.bitOr</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return the bitwise OR of the integers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-bitXor'>
-    <term><function>builtins.bitXor</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return the bitwise XOR of the integers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-builtins'>
-    <term><varname>builtins</varname></term>
-
-    <listitem><para>The set <varname>builtins</varname> contains all
-    the built-in functions and values.  You can use
-    <varname>builtins</varname> to test for the availability of
-    features in the Nix installation, e.g.,
-
-<programlisting>
-if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
-
-    This allows a Nix expression to fall back gracefully on older Nix
-    installations that don’t have the desired built-in
-    function.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-compareVersions'>
-    <term><function>builtins.compareVersions</function>
-    <emphasis>s1</emphasis> <emphasis>s2</emphasis></term>
-
-    <listitem><para>Compare two strings representing versions and
-    return <literal>-1</literal> if version
-    <emphasis>s1</emphasis> is older than version
-    <emphasis>s2</emphasis>, <literal>0</literal> if they are
-    the same, and <literal>1</literal> if
-    <emphasis>s1</emphasis> is newer than
-    <emphasis>s2</emphasis>.  The version comparison algorithm
-    is the same as the one used by <link
-    linkend="ssec-version-comparisons"><command>nix-env
-    -u</command></link>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-concatLists'>
-    <term><function>builtins.concatLists</function>
-    <emphasis>lists</emphasis></term>
-
-    <listitem><para>Concatenate a list of lists into a single
-    list.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-concatStringsSep'>
-    <term><function>builtins.concatStringsSep</function>
-    <emphasis>separator</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Concatenate a list of strings with a separator
-    between each element, e.g. <literal>concatStringsSep "/"
-    ["usr" "local" "bin"] == "usr/local/bin"</literal></para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-currentSystem'>
-    <term><varname>builtins.currentSystem</varname></term>
-
-    <listitem><para>The built-in value <varname>currentSystem</varname>
-    evaluates to the Nix platform identifier for the Nix installation
-    on which the expression is being evaluated, such as
-    <literal>"i686-linux"</literal> or
-    <literal>"x86_64-darwin"</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <!--
-  <varlistentry><term><function>currentTime</function></term>
-
-    <listitem><para>The built-in value <varname>currentTime</varname>
-    returns the current system time in seconds since 00:00:00 1/1/1970
-    UTC.  Due to the evaluation model of Nix expressions
-    (<emphasis>maximal laziness</emphasis>), it always yields the same
-    value within an execution of Nix.</para></listitem>
-
-  </varlistentry>
-  -->
-
-
-  <!--
-  <varlistentry><term><function>dependencyClosure</function></term>
-
-    <listitem><para>TODO</para></listitem>
-
-  </varlistentry>
-  -->
-
-
-  <varlistentry xml:id='builtin-deepSeq'>
-    <term><function>builtins.deepSeq</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>This is like <literal>seq
-    <emphasis>e1</emphasis>
-    <emphasis>e2</emphasis></literal>, except that
-    <emphasis>e1</emphasis> is evaluated
-    <emphasis>deeply</emphasis>: if it’s a list or set, its elements
-    or attributes are also evaluated recursively.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-derivation'>
-    <term><function>derivation</function>
-    <emphasis>attrs</emphasis></term>
-    <term><function>builtins.derivation</function>
-    <emphasis>attrs</emphasis></term>
-
-    <listitem><para><function>derivation</function> is described in
-    <xref linkend='ssec-derivation' />.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-dirOf'>
-    <term><function>dirOf</function> <emphasis>s</emphasis></term>
-    <term><function>builtins.dirOf</function> <emphasis>s</emphasis></term>
-
-    <listitem><para>Return the directory part of the string
-    <emphasis>s</emphasis>, that is, everything before the final
-    slash in the string.  This is similar to the GNU
-    <command>dirname</command> command.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-div'>
-    <term><function>builtins.div</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return the quotient of the numbers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-elem'>
-    <term><function>builtins.elem</function>
-    <emphasis>x</emphasis> <emphasis>xs</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if a value equal to
-    <emphasis>x</emphasis> occurs in the list
-    <emphasis>xs</emphasis>, and <literal>false</literal>
-    otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-elemAt'>
-    <term><function>builtins.elemAt</function>
-    <emphasis>xs</emphasis> <emphasis>n</emphasis></term>
-
-    <listitem><para>Return element <emphasis>n</emphasis> from
-    the list <emphasis>xs</emphasis>.  Elements are counted
-    starting from 0.  A fatal error occurs if the index is out of
-    bounds.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-fetchurl'>
-    <term><function>builtins.fetchurl</function>
-    <emphasis>url</emphasis></term>
-
-    <listitem><para>Download the specified URL and return the path of
-    the downloaded file. This function is not available if <link
-    linkend="conf-restrict-eval">restricted evaluation mode</link> is
-    enabled.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-fetchTarball'>
-    <term><function>fetchTarball</function>
-    <emphasis>url</emphasis></term>
-    <term><function>builtins.fetchTarball</function>
-    <emphasis>url</emphasis></term>
-
-    <listitem><para>Download the specified URL, unpack it and return
-    the path of the unpacked tree. The file must be a tape archive
-    (<filename>.tar</filename>) compressed with
-    <literal>gzip</literal>, <literal>bzip2</literal> or
-    <literal>xz</literal>. The top-level path component of the files
-    in the tarball is removed, so it is best if the tarball contains a
-    single directory at top level. The typical use of the function is
-    to obtain external Nix expression dependencies, such as a
-    particular version of Nixpkgs, e.g.
-
-<programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
-
-stdenv.mkDerivation { … }
-</programlisting>
-    </para>
-
-    <para>The fetched tarball is cached for a certain amount of time
-    (1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>.
-    You can change the cache timeout either on the command line with
-    <option>--option tarball-ttl <emphasis>number of seconds</emphasis></option> or
-    in the Nix configuration file with this option:
-    <literal><xref linkend="conf-tarball-ttl" /> <emphasis>number of seconds to cache</emphasis></literal>.
-    </para>
-
-    <para>Note that when obtaining the hash with <varname>nix-prefetch-url
-    </varname> the option <varname>--unpack</varname> is required.
-    </para>
-
-    <para>This function can also verify the contents against a hash.
-    In that case, the function takes a set instead of a URL. The set
-    requires the attribute <varname>url</varname> and the attribute
-    <varname>sha256</varname>, e.g.
-
-<programlisting>
-with import (fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
-  sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
-}) {};
-
-stdenv.mkDerivation { … }
-</programlisting>
-
-    </para>
-
-    <para>This function is not available if <link
-    linkend="conf-restrict-eval">restricted evaluation mode</link> is
-    enabled.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-fetchGit'>
-    <term>
-      <function>builtins.fetchGit</function>
-      <emphasis>args</emphasis>
-    </term>
-
-    <listitem>
-      <para>
-        Fetch a path from git. <emphasis>args</emphasis> can be
-        a URL, in which case the HEAD of the repo at that URL is
-        fetched. Otherwise, it can be an attribute with the following
-        attributes (all except <varname>url</varname> optional):
-      </para>
-
-      <variablelist>
-        <varlistentry>
-          <term>url</term>
-          <listitem>
-            <para>
-              The URL of the repo.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>name</term>
-          <listitem>
-            <para>
-              The name of the directory the repo should be exported to
-              in the store. Defaults to the basename of the URL.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>rev</term>
-          <listitem>
-            <para>
-              The git revision to fetch. Defaults to the tip of
-              <varname>ref</varname>.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>ref</term>
-          <listitem>
-            <para>
-              The git ref to look for the requested revision under.
-              This is often a branch or tag name. Defaults to
-              <literal>HEAD</literal>.
-            </para>
-
-            <para>
-              By default, the <varname>ref</varname> value is prefixed
-              with <literal>refs/heads/</literal>. As of Nix 2.3.0
-              Nix will not prefix <literal>refs/heads/</literal> if
-              <varname>ref</varname> starts with <literal>refs/</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>submodules</term>
-          <listitem>
-            <para>
-              A Boolean parameter that specifies whether submodules
-              should be checked out. Defaults to
-              <literal>false</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-      </variablelist>
-
-      <para>Here are some examples of how to use
-      <literal>fetchGit</literal>.</para>
-
-      <itemizedlist>
-
-        <listitem>
-          <para>To fetch a private repository over SSH:</para>
-
-          <programlisting>builtins.fetchGit {
-  url = "git@github.com:my-secret/repository.git";
-  ref = "master";
-  rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
-}</programlisting>
-
-        </listitem>
-
-        <listitem>
-          <para>To fetch an arbitrary reference:</para>
-
-          <programlisting>builtins.fetchGit {
-  url = "https://github.com/NixOS/nix.git";
-  ref = "refs/heads/0.5-release";
-}</programlisting>
-        </listitem>
-
-        <listitem>
-          <para>
-            If the revision you're looking for is in the default branch
-            of the git repository you don't strictly need to specify
-            the branch name in the <varname>ref</varname> attribute.
-          </para>
-          <para>
-            However, if the revision you're looking for is in a future
-            branch for the non-default branch you will need to specify
-            the the <varname>ref</varname> attribute as well.
-          </para>
-
-          <programlisting>builtins.fetchGit {
-  url = "https://github.com/nixos/nix.git";
-  rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-  ref = "1.11-maintenance";
-}</programlisting>
-
-          <note>
-            <para>
-              It is nice to always specify the branch which a revision
-              belongs to. Without the branch being specified, the
-              fetcher might fail if the default branch changes.
-              Additionally, it can be confusing to try a commit from a
-              non-default branch and see the fetch fail. If the branch
-              is specified the fault is much more obvious.
-            </para>
-          </note>
-        </listitem>
-
-        <listitem>
-          <para>
-            If the revision you're looking for is in the default branch
-            of the git repository you may omit the
-            <varname>ref</varname> attribute.
-          </para>
-          <programlisting>builtins.fetchGit {
-  url = "https://github.com/nixos/nix.git";
-  rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-}</programlisting>
-        </listitem>
-
-        <listitem>
-          <para>To fetch a specific tag:</para>
-          <programlisting>builtins.fetchGit {
-  url = "https://github.com/nixos/nix.git";
-  ref = "refs/tags/1.9";
-}</programlisting>
-        </listitem>
-
-        <listitem>
-          <para>To fetch the latest version of a remote branch:</para>
-        <programlisting>builtins.fetchGit {
-  url = "ssh://git@github.com/nixos/nix.git";
-  ref = "master";
-}</programlisting>
-          <note><para>Nix will refetch the branch in accordance to
-          <xref linkend="conf-tarball-ttl" />.</para></note>
-          <note><para>This behavior is disabled in <emphasis>Pure
-          evaluation mode</emphasis>.</para></note>
-        </listitem>
-      </itemizedlist>
-    </listitem>
-  </varlistentry>
-
-
-  <varlistentry><term><function>builtins.filter</function>
-  <emphasis>f</emphasis> <emphasis>xs</emphasis></term>
-
-    <listitem><para>Return a list consisting of the elements of
-    <emphasis>xs</emphasis> for which the function
-    <emphasis>f</emphasis> returns
-    <literal>true</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-filterSource'>
-    <term><function>builtins.filterSource</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem>
-
-      <para>This function allows you to copy sources into the Nix
-      store while filtering certain files.  For instance, suppose that
-      you want to use the directory <filename>source-dir</filename> as
-      an input to a Nix expression, e.g.
-
-<programlisting>
-stdenv.mkDerivation {
-  ...
-  src = ./source-dir;
-}
-</programlisting>
-
-      However, if <filename>source-dir</filename> is a Subversion
-      working copy, then all those annoying <filename>.svn</filename>
-      subdirectories will also be copied to the store.  Worse, the
-      contents of those directories may change a lot, causing lots of
-      spurious rebuilds.  With <function>filterSource</function> you
-      can filter out the <filename>.svn</filename> directories:
-
-<programlisting>
-  src = builtins.filterSource
-    (path: type: type != "directory" || baseNameOf path != ".svn")
-    ./source-dir;
-</programlisting>
-
-      </para>
-
-      <para>Thus, the first argument <emphasis>e1</emphasis>
-      must be a predicate function that is called for each regular
-      file, directory or symlink in the source tree
-      <emphasis>e2</emphasis>.  If the function returns
-      <literal>true</literal>, the file is copied to the Nix store,
-      otherwise it is omitted.  The function is called with two
-      arguments.  The first is the full path of the file.  The second
-      is a string that identifies the type of the file, which is
-      either <literal>"regular"</literal>,
-      <literal>"directory"</literal>, <literal>"symlink"</literal> or
-      <literal>"unknown"</literal> (for other kinds of files such as
-      device nodes or fifos — but note that those cannot be copied to
-      the Nix store, so if the predicate returns
-      <literal>true</literal> for them, the copy will fail). If you
-      exclude a directory, the entire corresponding subtree of
-      <emphasis>e2</emphasis> will be excluded.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-foldl-prime'>
-    <term><function>builtins.foldl’</function>
-    <emphasis>op</emphasis> <emphasis>nul</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Reduce a list by applying a binary operator, from
-    left to right, e.g. <literal>foldl’ op nul [x0 x1 x2 ...] = op (op
-    (op nul x0) x1) x2) ...</literal>. The operator is applied
-    strictly, i.e., its arguments are evaluated first. For example,
-    <literal>foldl’ (x: y: x + y) 0 [1 2 3]</literal> evaluates to
-    6.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-functionArgs'>
-    <term><function>builtins.functionArgs</function>
-    <emphasis>f</emphasis></term>
-
-    <listitem><para>
-    Return a set containing the names of the formal arguments expected
-    by the function <emphasis>f</emphasis>.
-    The value of each attribute is a Boolean denoting whether the corresponding
-    argument has a default value.  For instance,
-    <literal>functionArgs ({ x, y ? 123}: ...)  =  { x = false; y = true; }</literal>.
-    </para>
-
-    <para>"Formal argument" here refers to the attributes pattern-matched by
-    the function.  Plain lambdas are not included, e.g.
-    <literal>functionArgs (x: ...)  =  { }</literal>.
-    </para></listitem>
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-fromJSON'>
-    <term><function>builtins.fromJSON</function> <emphasis>e</emphasis></term>
-
-    <listitem><para>Convert a JSON string to a Nix
-    value. For example,
-
-<programlisting>
-builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
-</programlisting>
-
-    returns the value <literal>{ x = [ 1 2 3 ]; y = null;
-    }</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-genList'>
-    <term><function>builtins.genList</function>
-    <emphasis>generator</emphasis> <emphasis>length</emphasis></term>
-
-    <listitem><para>Generate list of size
-    <emphasis>length</emphasis>, with each element
-    <emphasis>i</emphasis> equal to the value returned by
-    <emphasis>generator</emphasis> <literal>i</literal>. For
-    example,
-
-<programlisting>
-builtins.genList (x: x * x) 5
-</programlisting>
-
-    returns the list <literal>[ 0 1 4 9 16 ]</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-getAttr'>
-    <term><function>builtins.getAttr</function>
-    <emphasis>s</emphasis> <emphasis>set</emphasis></term>
-
-    <listitem><para><function>getAttr</function> returns the attribute
-    named <emphasis>s</emphasis> from
-    <emphasis>set</emphasis>.  Evaluation aborts if the
-    attribute doesn’t exist.  This is a dynamic version of the
-    <literal>.</literal> operator, since <emphasis>s</emphasis>
-    is an expression rather than an identifier.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-getEnv'>
-    <term><function>builtins.getEnv</function>
-    <emphasis>s</emphasis></term>
-
-    <listitem><para><function>getEnv</function> returns the value of
-    the environment variable <emphasis>s</emphasis>, or an empty
-    string if the variable doesn’t exist.  This function should be
-    used with care, as it can introduce all sorts of nasty environment
-    dependencies in your Nix expression.</para>
-
-    <para><function>getEnv</function> is used in Nix Packages to
-    locate the file <filename>~/.nixpkgs/config.nix</filename>, which
-    contains user-local settings for Nix Packages.  (That is, it does
-    a <literal>getEnv "HOME"</literal> to locate the user’s home
-    directory.)</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-hasAttr'>
-    <term><function>builtins.hasAttr</function>
-    <emphasis>s</emphasis> <emphasis>set</emphasis></term>
-
-    <listitem><para><function>hasAttr</function> returns
-    <literal>true</literal> if <emphasis>set</emphasis> has an
-    attribute named <emphasis>s</emphasis>, and
-    <literal>false</literal> otherwise.  This is a dynamic version of
-    the <literal>?</literal>  operator, since
-    <emphasis>s</emphasis> is an expression rather than an
-    identifier.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-hashString'>
-    <term><function>builtins.hashString</function>
-    <emphasis>type</emphasis> <emphasis>s</emphasis></term>
-
-    <listitem><para>Return a base-16 representation of the
-    cryptographic hash of string <emphasis>s</emphasis>.  The
-    hash algorithm specified by <emphasis>type</emphasis> must
-    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
-    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-hashFile'>
-    <term><function>builtins.hashFile</function>
-    <emphasis>type</emphasis> <emphasis>p</emphasis></term>
-
-    <listitem><para>Return a base-16 representation of the
-    cryptographic hash of the file at path <emphasis>p</emphasis>.  The
-    hash algorithm specified by <emphasis>type</emphasis> must
-    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
-    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-head'>
-    <term><function>builtins.head</function>
-    <emphasis>list</emphasis></term>
-
-    <listitem><para>Return the first element of a list; abort
-    evaluation if the argument isn’t a list or is an empty list.  You
-    can test whether a list is empty by comparing it with
-    <literal>[]</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-import'>
-    <term><function>import</function>
-    <emphasis>path</emphasis></term>
-    <term><function>builtins.import</function>
-    <emphasis>path</emphasis></term>
-
-    <listitem><para>Load, parse and return the Nix expression in the
-    file <emphasis>path</emphasis>.  If <emphasis>path
-    </emphasis> is a directory, the file <filename>default.nix
-    </filename> in that directory is loaded.  Evaluation aborts if the
-    file doesn’t exist or contains an incorrect Nix expression.
-    <function>import</function> implements Nix’s module system: you
-    can put any Nix expression (such as a set or a function) in a
-    separate file, and use it from Nix expressions in other
-    files.</para>
-
-    <note><para>Unlike some languages, <function>import</function> is a regular
-    function in Nix. Paths using the angle bracket syntax (e.g., <function>
-    import</function> <emphasis>&lt;foo&gt;</emphasis>) are normal path
-    values (see <xref linkend='ssec-values' />).</para></note>
-
-    <para>A Nix expression loaded by <function>import</function> must
-    not contain any <emphasis>free variables</emphasis> (identifiers
-    that are not defined in the Nix expression itself and are not
-    built-in).  Therefore, it cannot refer to variables that are in
-    scope at the call site.  For instance, if you have a calling
-    expression
-
-<programlisting>
-rec {
-  x = 123;
-  y = import ./foo.nix;
-}</programlisting>
-
-    then the following <filename>foo.nix</filename> will give an
-    error:
-
-<programlisting>
-x + 456</programlisting>
-
-    since <varname>x</varname> is not in scope in
-    <filename>foo.nix</filename>.  If you want <varname>x</varname>
-    to be available in <filename>foo.nix</filename>, you should pass
-    it as a function argument:
-
-<programlisting>
-rec {
-  x = 123;
-  y = import ./foo.nix x;
-}</programlisting>
-
-    and
-
-<programlisting>
-x: x + 456</programlisting>
-
-    (The function argument doesn’t have to be called
-    <varname>x</varname> in <filename>foo.nix</filename>; any name
-    would work.)</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-intersectAttrs'>
-    <term><function>builtins.intersectAttrs</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return a set consisting of the attributes in the
-    set <emphasis>e2</emphasis> that also exist in the set
-    <emphasis>e1</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isAttrs'>
-    <term><function>builtins.isAttrs</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a set, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isList'>
-    <term><function>builtins.isList</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a list, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function>
-  <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a function, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isString'>
-    <term><function>builtins.isString</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a string, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isInt'>
-    <term><function>builtins.isInt</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to an int, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isFloat'>
-    <term><function>builtins.isFloat</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a float, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-isBool'>
-    <term><function>builtins.isBool</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a bool, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><function>builtins.isPath</function>
-  <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to a path, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-isNull'>
-    <term><function>isNull</function>
-    <emphasis>e</emphasis></term>
-    <term><function>builtins.isNull</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if
-    <emphasis>e</emphasis> evaluates to <literal>null</literal>,
-    and <literal>false</literal> otherwise.</para>
-
-    <warning><para>This function is <emphasis>deprecated</emphasis>;
-    just write <literal>e == null</literal> instead.</para></warning>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-length'>
-    <term><function>builtins.length</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return the length of the list
-    <emphasis>e</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-lessThan'>
-    <term><function>builtins.lessThan</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if the number
-    <emphasis>e1</emphasis> is less than the number
-    <emphasis>e2</emphasis>, and <literal>false</literal>
-    otherwise.  Evaluation aborts if either
-    <emphasis>e1</emphasis> or <emphasis>e2</emphasis>
-    does not evaluate to a number.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-listToAttrs'>
-    <term><function>builtins.listToAttrs</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Construct a set from a list specifying the names
-    and values of each attribute.  Each element of the list should be
-    a set consisting of a string-valued attribute
-    <varname>name</varname> specifying the name of the attribute, and
-    an attribute <varname>value</varname> specifying its value.
-    Example:
-
-<programlisting>
-builtins.listToAttrs
-  [ { name = "foo"; value = 123; }
-    { name = "bar"; value = 456; }
-  ]
-</programlisting>
-
-    evaluates to
-
-<programlisting>
-{ foo = 123; bar = 456; }
-</programlisting>
-
-    </para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-map'>
-    <term><function>map</function>
-    <emphasis>f</emphasis> <emphasis>list</emphasis></term>
-    <term><function>builtins.map</function>
-    <emphasis>f</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Apply the function <emphasis>f</emphasis> to
-    each element in the list <emphasis>list</emphasis>.  For
-    example,
-
-<programlisting>
-map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting>
-
-    evaluates to <literal>[ "foobar" "foobla" "fooabc"
-    ]</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-match'>
-    <term><function>builtins.match</function>
-    <emphasis>regex</emphasis> <emphasis>str</emphasis></term>
-
-    <listitem><para>Returns a list if the <link
-    xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-    POSIX regular expression</link> <emphasis>regex</emphasis>
-    matches <emphasis>str</emphasis> precisely, otherwise returns
-    <literal>null</literal>.  Each item in the list is a regex group.
-
-<programlisting>
-builtins.match "ab" "abc"
-</programlisting>
-
-Evaluates to <literal>null</literal>.
-
-<programlisting>
-builtins.match "abc" "abc"
-</programlisting>
-
-Evaluates to <literal>[ ]</literal>.
-
-<programlisting>
-builtins.match "a(b)(c)" "abc"
-</programlisting>
-
-Evaluates to <literal>[ "b" "c" ]</literal>.
-
-<programlisting>
-builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
-</programlisting>
-
-Evaluates to <literal>[ "foo" ]</literal>.
-
-    </para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-mul'>
-    <term><function>builtins.mul</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return the product of the numbers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-parseDrvName'>
-    <term><function>builtins.parseDrvName</function>
-    <emphasis>s</emphasis></term>
-
-    <listitem><para>Split the string <emphasis>s</emphasis> into
-    a package name and version.  The package name is everything up to
-    but not including the first dash followed by a digit, and the
-    version is everything following that dash.  The result is returned
-    in a set <literal>{ name, version }</literal>.  Thus,
-    <literal>builtins.parseDrvName "nix-0.12pre12876"</literal>
-    returns <literal>{ name = "nix"; version = "0.12pre12876";
-    }</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-path'>
-    <term>
-      <function>builtins.path</function>
-      <emphasis>args</emphasis>
-    </term>
-
-    <listitem>
-      <para>
-        An enrichment of the built-in path type, based on the attributes
-        present in <emphasis>args</emphasis>. All are optional
-        except <varname>path</varname>:
-      </para>
-
-      <variablelist>
-        <varlistentry>
-          <term>path</term>
-          <listitem>
-            <para>The underlying path.</para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>name</term>
-          <listitem>
-            <para>
-              The name of the path when added to the store. This can
-              used to reference paths that have nix-illegal characters
-              in their names, like <literal>@</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>filter</term>
-          <listitem>
-            <para>
-              A function of the type expected by
-              <link linkend="builtin-filterSource">builtins.filterSource</link>,
-              with the same semantics.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>recursive</term>
-          <listitem>
-            <para>
-              When <literal>false</literal>, when
-              <varname>path</varname> is added to the store it is with a
-              flat hash, rather than a hash of the NAR serialization of
-              the file. Thus, <varname>path</varname> must refer to a
-              regular file, not a directory. This allows similar
-              behavior to <literal>fetchurl</literal>. Defaults to
-              <literal>true</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>sha256</term>
-          <listitem>
-            <para>
-              When provided, this is the expected hash of the file at
-              the path. Evaluation will fail if the hash is incorrect,
-              and providing a hash allows
-              <literal>builtins.path</literal> to be used even when the
-              <literal>pure-eval</literal> nix config option is on.
-            </para>
-          </listitem>
-        </varlistentry>
-      </variablelist>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-pathExists'>
-    <term><function>builtins.pathExists</function>
-    <emphasis>path</emphasis></term>
-
-    <listitem><para>Return <literal>true</literal> if the path
-    <emphasis>path</emphasis> exists at evaluation time, and
-    <literal>false</literal> otherwise.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-placeholder'>
-    <term><function>builtins.placeholder</function>
-    <emphasis>output</emphasis></term>
-
-    <listitem><para>Return a placeholder string for the specified
-    <emphasis>output</emphasis> that will be substituted by the
-    corresponding output path at build time. Typical outputs would be
-    <literal>"out"</literal>, <literal>"bin"</literal> or
-    <literal>"dev"</literal>.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-readDir'>
-    <term><function>builtins.readDir</function>
-    <emphasis>path</emphasis></term>
-
-    <listitem><para>Return the contents of the directory
-    <emphasis>path</emphasis> as a set mapping directory entries
-    to the corresponding file type. For instance, if directory
-    <filename>A</filename> contains a regular file
-    <filename>B</filename> and another directory
-    <filename>C</filename>, then <literal>builtins.readDir
-    ./A</literal> will return the set
-
-<programlisting>
-{ B = "regular"; C = "directory"; }</programlisting>
-
-    The possible values for the file type are
-    <literal>"regular"</literal>, <literal>"directory"</literal>,
-    <literal>"symlink"</literal> and
-    <literal>"unknown"</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-readFile'>
-    <term><function>builtins.readFile</function>
-    <emphasis>path</emphasis></term>
-
-    <listitem><para>Return the contents of the file
-    <emphasis>path</emphasis> as a string.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-removeAttrs'>
-    <term><function>removeAttrs</function>
-    <emphasis>set</emphasis> <emphasis>list</emphasis></term>
-    <term><function>builtins.removeAttrs</function>
-    <emphasis>set</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Remove the attributes listed in
-    <emphasis>list</emphasis> from
-    <emphasis>set</emphasis>.  The attributes don’t have to
-    exist in <emphasis>set</emphasis>. For instance,
-
-<programlisting>
-removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting>
-
-    evaluates to <literal>{ y = 2; }</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-replaceStrings'>
-    <term><function>builtins.replaceStrings</function>
-    <emphasis>from</emphasis> <emphasis>to</emphasis> <emphasis>s</emphasis></term>
-
-    <listitem><para>Given string <emphasis>s</emphasis>, replace
-    every occurrence of the strings in <emphasis>from</emphasis>
-    with the corresponding string in
-    <emphasis>to</emphasis>. For example,
-
-<programlisting>
-builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
-</programlisting>
-
-    evaluates to <literal>"fabir"</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-seq'>
-    <term><function>builtins.seq</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Evaluate <emphasis>e1</emphasis>, then
-    evaluate and return <emphasis>e2</emphasis>. This ensures
-    that a computation is strict in the value of
-    <emphasis>e1</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-sort'>
-    <term><function>builtins.sort</function>
-    <emphasis>comparator</emphasis> <emphasis>list</emphasis></term>
-
-    <listitem><para>Return <emphasis>list</emphasis> in sorted
-    order. It repeatedly calls the function
-    <emphasis>comparator</emphasis> with two elements. The
-    comparator should return <literal>true</literal> if the first
-    element is less than the second, and <literal>false</literal>
-    otherwise. For example,
-
-<programlisting>
-builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
-</programlisting>
-
-    produces the list <literal>[ 42 77 147 249 483 526
-    ]</literal>.</para>
-
-    <para>This is a stable sort: it preserves the relative order of
-    elements deemed equal by the comparator.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-split'>
-    <term><function>builtins.split</function>
-    <emphasis>regex</emphasis> <emphasis>str</emphasis></term>
-
-    <listitem><para>Returns a list composed of non matched strings interleaved
-    with the lists of the <link
-    xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-    POSIX regular expression</link> <emphasis>regex</emphasis> matches
-    of <emphasis>str</emphasis>. Each item in the lists of matched
-    sequences is a regex group.
-
-<programlisting>
-builtins.split "(a)b" "abc"
-</programlisting>
-
-Evaluates to <literal>[ "" [ "a" ] "c" ]</literal>.
-
-<programlisting>
-builtins.split "([ac])" "abc"
-</programlisting>
-
-Evaluates to <literal>[ "" [ "a" ] "b" [ "c" ] "" ]</literal>.
-
-<programlisting>
-builtins.split "(a)|(c)" "abc"
-</programlisting>
-
-Evaluates to <literal>[ "" [ "a" null ] "b" [ null "c" ] "" ]</literal>.
-
-<programlisting>
-builtins.split "([[:upper:]]+)" "  FOO   "
-</programlisting>
-
-Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
-
-    </para></listitem>
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-splitVersion'>
-    <term><function>builtins.splitVersion</function>
-    <emphasis>s</emphasis></term>
-
-    <listitem><para>Split a string representing a version into its
-    components, by the same version splitting logic underlying the
-    version comparison in <link linkend="ssec-version-comparisons">
-    <command>nix-env -u</command></link>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-stringLength'>
-    <term><function>builtins.stringLength</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return the length of the string
-    <emphasis>e</emphasis>.  If <emphasis>e</emphasis> is
-    not a string, evaluation is aborted.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-sub'>
-    <term><function>builtins.sub</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Return the difference between the numbers
-    <emphasis>e1</emphasis> and
-    <emphasis>e2</emphasis>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-substring'>
-    <term><function>builtins.substring</function>
-    <emphasis>start</emphasis> <emphasis>len</emphasis>
-    <emphasis>s</emphasis></term>
-
-    <listitem><para>Return the substring of
-    <emphasis>s</emphasis> from character position
-    <emphasis>start</emphasis> (zero-based) up to but not
-    including <emphasis>start + len</emphasis>.  If
-    <emphasis>start</emphasis> is greater than the length of the
-    string, an empty string is returned, and if <emphasis>start +
-    len</emphasis> lies beyond the end of the string, only the
-    substring up to the end of the string is returned.
-    <emphasis>start</emphasis> must be
-    non-negative. For example,
-
-<programlisting>
-builtins.substring 0 3 "nixos"
-</programlisting>
-
-   evaluates to <literal>"nix"</literal>.
-   </para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-tail'>
-    <term><function>builtins.tail</function>
-    <emphasis>list</emphasis></term>
-
-    <listitem><para>Return the second to last elements of a list;
-    abort evaluation if the argument isn’t a list or is an empty
-    list.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-throw'>
-    <term><function>throw</function>
-    <emphasis>s</emphasis></term>
-    <term><function>builtins.throw</function>
-    <emphasis>s</emphasis></term>
-
-    <listitem><para>Throw an error message
-    <emphasis>s</emphasis>.  This usually aborts Nix expression
-    evaluation, but in <command>nix-env -qa</command> and other
-    commands that try to evaluate a set of derivations to get
-    information about those derivations, a derivation that throws an
-    error is silently skipped (which is not the case for
-    <function>abort</function>).</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-toFile'>
-    <term><function>builtins.toFile</function>
-    <emphasis>name</emphasis>
-    <emphasis>s</emphasis></term>
-
-    <listitem><para>Store the string <emphasis>s</emphasis> in a
-    file in the Nix store and return its path.  The file has suffix
-    <emphasis>name</emphasis>.  This file can be used as an
-    input to derivations.  One application is to write builders
-    “inline”.  For instance, the following Nix expression combines
-    <xref linkend='ex-hello-nix' /> and <xref
-    linkend='ex-hello-builder' /> into one file:
-
-<programlisting>
-{ stdenv, fetchurl, perl }:
-
-stdenv.mkDerivation {
-  name = "hello-2.1.1";
-
-  builder = builtins.toFile "builder.sh" "
-    source $stdenv/setup
-
-    PATH=$perl/bin:$PATH
-
-    tar xvfz $src
-    cd hello-*
-    ./configure --prefix=$out
-    make
-    make install
-  ";
-
-  src = fetchurl {
-    url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-    sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-  };
-  inherit perl;
-}</programlisting>
-
-    </para>
-
-    <para>It is even possible for one file to refer to another, e.g.,
-
-<programlisting>
-  builder = let
-    configFile = builtins.toFile "foo.conf" "
-      # This is some dummy configuration file.
-      <emphasis>...</emphasis>
-    ";
-  in builtins.toFile "builder.sh" "
-    source $stdenv/setup
-    <emphasis>...</emphasis>
-    cp ${configFile} $out/etc/foo.conf
-  ";</programlisting>
-
-    Note that <literal>${configFile}</literal> is an antiquotation
-    (see <xref linkend='ssec-values' />), so the result of the
-    expression <literal>configFile</literal> (i.e., a path like
-    <filename>/nix/store/m7p7jfny445k...-foo.conf</filename>) will be
-    spliced into the resulting string.</para>
-
-    <para>It is however <emphasis>not</emphasis> allowed to have files
-    mutually referring to each other, like so:
-
-<programlisting>
-let
-  foo = builtins.toFile "foo" "...${bar}...";
-  bar = builtins.toFile "bar" "...${foo}...";
-in foo</programlisting>
-
-    This is not allowed because it would cause a cyclic dependency in
-    the computation of the cryptographic hashes for
-    <varname>foo</varname> and <varname>bar</varname>.</para>
-    <para>It is also not possible to reference the result of a derivation.
-    If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to
-    do that.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-toJSON'>
-    <term><function>builtins.toJSON</function> <emphasis>e</emphasis></term>
-
-    <listitem><para>Return a string containing a JSON representation
-    of <emphasis>e</emphasis>.  Strings, integers, floats, booleans,
-    nulls and lists are mapped to their JSON equivalents.  Sets
-    (except derivations) are represented as objects.  Derivations are
-    translated to a JSON string containing the derivation’s output
-    path.  Paths are copied to the store and represented as a JSON
-    string of the resulting store path.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-toPath'>
-    <term><function>builtins.toPath</function> <emphasis>s</emphasis></term>
-
-    <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal>
-    to convert a string into an absolute path. For relative paths,
-    use <literal>./. + "/path"</literal>.
-    </para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-toString'>
-    <term><function>toString</function> <emphasis>e</emphasis></term>
-    <term><function>builtins.toString</function> <emphasis>e</emphasis></term>
-
-    <listitem><para>Convert the expression
-    <emphasis>e</emphasis> to a string.
-    <emphasis>e</emphasis> can be:</para>
-    <itemizedlist>
-      <listitem><para>A string (in which case the string is returned unmodified).</para></listitem>
-      <listitem><para>A path (e.g., <literal>toString /foo/bar</literal> yields <literal>"/foo/bar"</literal>.</para></listitem>
-      <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem>
-      <listitem><para>An integer.</para></listitem>
-      <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem>
-      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>).</para></listitem>
-      <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem>
-    </itemizedlist>
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-toXML'>
-    <term><function>builtins.toXML</function> <emphasis>e</emphasis></term>
-
-    <listitem><para>Return a string containing an XML representation
-    of <emphasis>e</emphasis>.  The main application for
-    <function>toXML</function> is to communicate information with the
-    builder in a more structured format than plain environment
-    variables.</para>
-
-    <!-- TODO: more formally describe the schema of the XML
-    representation -->
-
-    <para>Here is an example where this is
-    the case:
-    </para>
-
-<programlisting><![CDATA[
-{ stdenv, fetchurl, libxslt, jira, uberwiki }:
-
-stdenv.mkDerivation (rec {
-  name = "web-server";
-
-  buildInputs = [ libxslt ];
-
-  builder = builtins.toFile "builder.sh" "
-    source $stdenv/setup
-    mkdir $out
-    echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> ① <![CDATA[
-  ";
-
-  stylesheet = builtins.toFile "stylesheet.xsl"]]> ② <![CDATA[
-   "<?xml version='1.0' encoding='UTF-8'?>
-    <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
-      <xsl:template match='/'>
-        <Configure>
-          <xsl:for-each select='/expr/list/attrs'>
-            <Call name='addWebApplication'>
-              <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg>
-              <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg>
-            </Call>
-          </xsl:for-each>
-        </Configure>
-      </xsl:template>
-    </xsl:stylesheet>
-  ";
-
-  servlets = builtins.toXML []]> ③ <![CDATA[
-    { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
-    { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
-  ];
-})]]></programlisting>
-
-    <para>The builder is supposed to generate the configuration file
-    for a <link xlink:href='http://jetty.mortbay.org/'>Jetty servlet
-    container</link>.  A servlet container contains a number of
-    servlets (<filename>*.war</filename> files) each exported under a
-    specific URI prefix.  So the servlet configuration is a list of
-    sets containing the <varname>path</varname> and
-    <varname>war</varname> of the servlet (<xref
-    linkend='ex-toxml-co-servlets' />).  This kind of information is
-    difficult to communicate with the normal method of passing
-    information through an environment variable, which just
-    concatenates everything together into a string (which might just
-    work in this case, but wouldn’t work if fields are optional or
-    contain lists themselves).  Instead the Nix expression is
-    converted to an XML representation with
-    <function>toXML</function>, which is unambiguous and can easily be
-    processed with the appropriate tools.  For instance, in the
-    example an XSLT stylesheet (at point ②) is applied to it (at point
-    ①) to generate the XML configuration file for the Jetty server.
-    The XML representation produced at point ③ by
-    <function>toXML</function> is as follows:</para>
-
-<programlisting><![CDATA[<?xml version='1.0' encoding='utf-8'?>
-<expr>
-  <list>
-    <attrs>
-      <attr name="path">
-        <string value="/bugtracker" />
-      </attr>
-      <attr name="war">
-        <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="path">
-        <string value="/wiki" />
-      </attr>
-      <attr name="war">
-        <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" />
-      </attr>
-    </attrs>
-  </list>
-</expr>]]></programlisting>
-
-    <para>Note that <xref linkend='ex-toxml' /> uses the <function
-    linkend='builtin-toFile'>toFile</function> built-in to write the
-    builder and the stylesheet “inline” in the Nix expression.  The
-    path of the stylesheet is spliced into the builder using the
-    syntax <literal>xsltproc ${stylesheet}</literal>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-trace'>
-    <term><function>builtins.trace</function>
-    <emphasis>e1</emphasis> <emphasis>e2</emphasis></term>
-
-    <listitem><para>Evaluate <emphasis>e1</emphasis> and print its
-    abstract syntax representation on standard error.  Then return
-    <emphasis>e2</emphasis>.  This function is useful for
-    debugging.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id='builtin-tryEval'>
-    <term><function>builtins.tryEval</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Try to shallowly evaluate <emphasis>e</emphasis>.
-    Return a set containing the attributes <literal>success</literal>
-    (<literal>true</literal> if <emphasis>e</emphasis> evaluated
-    successfully, <literal>false</literal> if an error was thrown) and
-    <literal>value</literal>, equalling <emphasis>e</emphasis>
-    if successful and <literal>false</literal> otherwise. Note that this
-    doesn't evaluate <emphasis>e</emphasis> deeply, so
-    <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success
-    </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq
-    </literal> one can get the expected result: <literal>let e = { x = throw "";
-    }; in (builtins.tryEval (builtins.deepSeq e e)).success</literal> will be
-    <literal>false</literal>.
-    </para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id='builtin-typeOf'>
-    <term><function>builtins.typeOf</function>
-    <emphasis>e</emphasis></term>
-
-    <listitem><para>Return a string representing the type of the value
-    <emphasis>e</emphasis>, namely <literal>"int"</literal>,
-    <literal>"bool"</literal>, <literal>"string"</literal>,
-    <literal>"path"</literal>, <literal>"null"</literal>,
-    <literal>"set"</literal>, <literal>"list"</literal>,
-    <literal>"lambda"</literal> or
-    <literal>"float"</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-</variablelist>
-
-
-</section>
diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml
deleted file mode 100644
index 0625bcdfe..000000000
--- a/doc/manual/expressions/derivations.xml
+++ /dev/null
@@ -1,210 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-derivation">
-
-<title>Derivations</title>
-
-<para>The most important built-in function is
-<function>derivation</function>, which is used to describe a single
-derivation (a build action).  It takes as input a set, the attributes
-of which specify the inputs of the build.</para>
-
-<itemizedlist>
-
-  <listitem xml:id="attr-system"><para>There must be an attribute
-  named <varname>system</varname> whose value must be a string
-  specifying a Nix system type, such as
-  <literal>"i686-linux"</literal> or
-  <literal>"x86_64-darwin"</literal>. (To figure out your system type,
-  run <literal>nix -vv --version</literal>.) The build can only be
-  performed on a machine and operating system matching the system
-  type.  (Nix can automatically forward builds for other platforms by
-  forwarding them to other machines; see <xref
-  linkend='chap-distributed-builds' />.)</para></listitem>
-
-  <listitem><para>There must be an attribute named
-  <varname>name</varname> whose value must be a string.  This is used
-  as a symbolic name for the package by <command>nix-env</command>,
-  and it is appended to the output paths of the
-  derivation.</para></listitem>
-
-  <listitem><para>There must be an attribute named
-  <varname>builder</varname> that identifies the program that is
-  executed to perform the build.  It can be either a derivation or a
-  source (a local file reference, e.g.,
-  <filename>./builder.sh</filename>).</para></listitem>
-
-  <listitem><para>Every attribute is passed as an environment variable
-  to the builder.  Attribute values are translated to environment
-  variables as follows:
-
-    <itemizedlist>
-
-      <listitem><para>Strings and numbers are just passed
-      verbatim.</para></listitem>
-
-      <listitem><para>A <emphasis>path</emphasis> (e.g.,
-      <filename>../foo/sources.tar</filename>) causes the referenced
-      file to be copied to the store; its location in the store is put
-      in the environment variable.  The idea is that all sources
-      should reside in the Nix store, since all inputs to a derivation
-      should reside in the Nix store.</para></listitem>
-
-      <listitem><para>A <emphasis>derivation</emphasis> causes that
-      derivation to be built prior to the present derivation; its
-      default output path is put in the environment
-      variable.</para></listitem>
-
-      <listitem><para>Lists of the previous types are also allowed.
-      They are simply concatenated, separated by
-      spaces.</para></listitem>
-
-      <listitem><para><literal>true</literal> is passed as the string
-      <literal>1</literal>, <literal>false</literal> and
-      <literal>null</literal> are passed as an empty string.
-      </para></listitem>
-    </itemizedlist>
-
-  </para></listitem>
-
-  <listitem><para>The optional attribute <varname>args</varname>
-  specifies command-line arguments to be passed to the builder.  It
-  should be a list.</para></listitem>
-
-  <listitem><para>The optional attribute <varname>outputs</varname>
-  specifies a list of symbolic outputs of the derivation.  By default,
-  a derivation produces a single output path, denoted as
-  <literal>out</literal>.  However, derivations can produce multiple
-  output paths.  This is useful because it allows outputs to be
-  downloaded or garbage-collected separately.  For instance, imagine a
-  library package that provides a dynamic library, header files, and
-  documentation.  A program that links against the library doesn’t
-  need the header files and documentation at runtime, and it doesn’t
-  need the documentation at build time.  Thus, the library package
-  could specify:
-<programlisting>
-outputs = [ "lib" "headers" "doc" ];
-</programlisting>
-  This will cause Nix to pass environment variables
-  <literal>lib</literal>, <literal>headers</literal> and
-  <literal>doc</literal> to the builder containing the intended store
-  paths of each output.  The builder would typically do something like
-<programlisting>
-./configure --libdir=$lib/lib --includedir=$headers/include --docdir=$doc/share/doc
-</programlisting>
-  for an Autoconf-style package.  You can refer to each output of a
-  derivation by selecting it as an attribute, e.g.
-<programlisting>
-buildInputs = [ pkg.lib pkg.headers ];
-</programlisting>
-  The first element of <varname>outputs</varname> determines the
-  <emphasis>default output</emphasis>.  Thus, you could also write
-<programlisting>
-buildInputs = [ pkg pkg.headers ];
-</programlisting>
-  since <literal>pkg</literal> is equivalent to
-  <literal>pkg.lib</literal>.</para></listitem>
-
-</itemizedlist>
-
-<para>The function <function>mkDerivation</function> in the Nixpkgs
-standard environment is a wrapper around
-<function>derivation</function> that adds a default value for
-<varname>system</varname> and always uses Bash as the builder, to
-which the supplied builder is passed as a command-line argument.  See
-the Nixpkgs manual for details.</para>
-
-<para>The builder is executed as follows:
-
-<itemizedlist>
-
-  <listitem><para>A temporary directory is created under the directory
-  specified by <literal>TMPDIR</literal> (default
-  <filename>/tmp</filename>) where the build will take place.  The
-  current directory is changed to this directory.</para></listitem>
-
-  <listitem><para>The environment is cleared and set to the derivation
-  attributes, as specified above.</para></listitem>
-
-  <listitem><para>In addition, the following variables are set:
-
-  <itemizedlist>
-
-    <listitem><para><literal>NIX_BUILD_TOP</literal> contains the path of
-    the temporary directory for this build.</para></listitem>
-
-    <listitem><para>Also, <literal>TMPDIR</literal>,
-    <literal>TEMPDIR</literal>, <literal>TMP</literal>, <literal>TEMP</literal>
-    are set to point to the temporary directory.  This is to prevent
-    the builder from accidentally writing temporary files anywhere
-    else.  Doing so might cause interference by other
-    processes.</para></listitem>
-
-    <listitem><para><literal>PATH</literal> is set to
-    <filename>/path-not-set</filename> to prevent shells from
-    initialising it to their built-in default value.</para></listitem>
-
-    <listitem><para><literal>HOME</literal> is set to
-    <filename>/homeless-shelter</filename> to prevent programs from
-    using <filename>/etc/passwd</filename> or the like to find the
-    user's home directory, which could cause impurity.  Usually, when
-    <literal>HOME</literal> is set, it is used as the location of the home
-    directory, even if it points to a non-existent
-    path.</para></listitem>
-
-    <listitem><para><literal>NIX_STORE</literal> is set to the path of the
-    top-level Nix store directory (typically,
-    <filename>/nix/store</filename>).</para></listitem>
-
-    <listitem><para>For each output declared in
-    <varname>outputs</varname>, the corresponding environment variable
-    is set to point to the intended path in the Nix store for that
-    output.  Each output path is a concatenation of the cryptographic
-    hash of all build inputs, the <varname>name</varname> attribute
-    and the output name.  (The output name is omitted if it’s
-    <literal>out</literal>.)</para></listitem>
-
-  </itemizedlist>
-
-  </para></listitem>
-
-  <listitem><para>If an output path already exists, it is removed.
-  Also, locks are acquired to prevent multiple Nix instances from
-  performing the same build at the same time.</para></listitem>
-
-  <listitem><para>A log of the combined standard output and error is
-  written to <filename>/nix/var/log/nix</filename>.</para></listitem>
-
-  <listitem><para>The builder is executed with the arguments specified
-  by the attribute <varname>args</varname>.  If it exits with exit
-  code 0, it is considered to have succeeded.</para></listitem>
-
-  <listitem><para>The temporary directory is removed (unless the
-  <option>-K</option> option was specified).</para></listitem>
-
-  <listitem><para>If the build was successful, Nix scans each output
-  path for references to input paths by looking for the hash parts of
-  the input paths.  Since these are potential runtime dependencies,
-  Nix registers them as dependencies of the output
-  paths.</para></listitem>
-
-  <listitem><para>After the build, Nix sets the last-modified
-  timestamp on all files in the build result to 1 (00:00:01 1/1/1970
-  UTC), sets the group to the default group, and sets the mode of the
-  file to 0444 or 0555 (i.e., read-only, with execute permission
-  enabled if the file was originally executable).  Note that possible
-  <literal>setuid</literal> and <literal>setgid</literal> bits are
-  cleared.  Setuid and setgid programs are not currently supported by
-  Nix.  This is because the Nix archives used in deployment have no
-  concept of ownership information, and because it makes the build
-  result dependent on the user performing the build.</para></listitem>
-
-</itemizedlist>
-
-</para>
-
-<xi:include href="advanced-attributes.xml" />
-
-</section>
diff --git a/doc/manual/expressions/expression-language.xml b/doc/manual/expressions/expression-language.xml
deleted file mode 100644
index 240ef80f1..000000000
--- a/doc/manual/expressions/expression-language.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-expression-language">
-
-<title>Nix Expression Language</title>
-
-<para>The Nix expression language is a pure, lazy, functional
-language.  Purity means that operations in the language don't have
-side-effects (for instance, there is no variable assignment).
-Laziness means that arguments to functions are evaluated only when
-they are needed.  Functional means that functions are
-<quote>normal</quote> values that can be passed around and manipulated
-in interesting ways.  The language is not a full-featured, general
-purpose language.  Its main job is to describe packages,
-compositions of packages, and the variability within
-packages.</para>
-
-<para>This section presents the various features of the
-language.</para>
-
-<xi:include href="language-values.xml" />
-<xi:include href="language-constructs.xml" />
-<xi:include href="language-operators.xml" />
-<xi:include href="derivations.xml" />
-<xi:include href="builtins.xml" />
-
-
-</chapter>
diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml
deleted file mode 100644
index 853e40b58..000000000
--- a/doc/manual/expressions/expression-syntax.xml
+++ /dev/null
@@ -1,146 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='sec-expression-syntax'>
-
-<title>Expression Syntax</title>
-
-<para>Here is a Nix expression for GNU Hello:</para>
-
-<programlisting>
-{ stdenv, fetchurl, perl }: ①
-
-stdenv.mkDerivation { ②
-  name = "hello-2.1.1"; ③
-  builder = ./builder.sh; ④
-  src = fetchurl { ⑤
-    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-    sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-  };
-  inherit perl; ⑥
-}</programlisting>
-
-<para>This file is actually already in the Nix Packages collection in
-<filename>pkgs/applications/misc/hello/ex-1/default.nix</filename>.
-It is customary to place each package in a separate directory and call
-the single Nix expression in that directory
-<filename>default.nix</filename>.  The file has the following elements
-(referenced from the figure by number):
-
-<orderedlist>
-
-  <listitem>
-
-    <para>This states that the expression is a
-    <emphasis>function</emphasis> that expects to be called with three
-    arguments: <varname>stdenv</varname>, <varname>fetchurl</varname>,
-    and <varname>perl</varname>.  They are needed to build Hello, but
-    we don't know how to build them here; that's why they are function
-    arguments.  <varname>stdenv</varname> is a package that is used
-    by almost all Nix Packages packages; it provides a
-    <quote>standard</quote> environment consisting of the things you
-    would expect in a basic Unix environment: a C/C++ compiler (GCC,
-    to be precise), the Bash shell, fundamental Unix tools such as
-    <command>cp</command>, <command>grep</command>,
-    <command>tar</command>, etc.  <varname>fetchurl</varname> is a
-    function that downloads files.  <varname>perl</varname> is the
-    Perl interpreter.</para>
-
-    <para>Nix functions generally have the form <literal>{ x, y, ...,
-    z }: e</literal> where <varname>x</varname>, <varname>y</varname>,
-    etc. are the names of the expected arguments, and where
-    <emphasis>e</emphasis> is the body of the function.  So
-    here, the entire remainder of the file is the body of the
-    function; when given the required arguments, the body should
-    describe how to build an instance of the Hello package.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>So we have to build a package.  Building something from
-    other stuff is called a <emphasis>derivation</emphasis> in Nix (as
-    opposed to sources, which are built by humans instead of
-    computers).  We perform a derivation by calling
-    <varname>stdenv.mkDerivation</varname>.
-    <varname>mkDerivation</varname> is a function provided by
-    <varname>stdenv</varname> that builds a package from a set of
-    <emphasis>attributes</emphasis>.  A set is just a list of
-    key/value pairs where each key is a string and each value is an
-    arbitrary Nix expression.  They take the general form <literal>{
-    <emphasis>name1</emphasis> =
-    <emphasis>expr1</emphasis>; <emphasis>...</emphasis>
-    <emphasis>nameN</emphasis> =
-    <emphasis>exprN</emphasis>; }</literal>.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>The attribute <varname>name</varname> specifies the symbolic
-    name and version of the package.  Nix doesn't really care about
-    these things, but they are used by for instance <command>nix-env
-    -q</command> to show a <quote>human-readable</quote> name for
-    packages.  This attribute is required by
-    <varname>mkDerivation</varname>.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>The attribute <varname>builder</varname> specifies the
-    builder.  This attribute can sometimes be omitted, in which case
-    <varname>mkDerivation</varname> will fill in a default builder
-    (which does a <literal>configure; make; make install</literal>, in
-    essence).  Hello is sufficiently simple that the default builder
-    would suffice, but in this case, we will show an actual builder
-    for educational purposes.  The value
-    <command>./builder.sh</command> refers to the shell script shown
-    in <xref linkend='ex-hello-builder' />, discussed below.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>The builder has to know what the sources of the package
-    are.  Here, the attribute <varname>src</varname> is bound to the
-    result of a call to the <command>fetchurl</command> function.
-    Given a URL and a SHA-256 hash of the expected contents of the file
-    at that URL, this function builds a derivation that downloads the
-    file and checks its hash.  So the sources are a dependency that
-    like all other dependencies is built before Hello itself is
-    built.</para>
-
-    <para>Instead of <varname>src</varname> any other name could have
-    been used, and in fact there can be any number of sources (bound
-    to different attributes).  However, <varname>src</varname> is
-    customary, and it's also expected by the default builder (which we
-    don't use in this example).</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Since the derivation requires Perl, we have to pass the
-    value of the <varname>perl</varname> function argument to the
-    builder.  All attributes in the set are actually passed as
-    environment variables to the builder, so declaring an attribute
-
-    <programlisting>
-perl = perl;</programlisting>
-
-    will do the trick: it binds an attribute <varname>perl</varname>
-    to the function argument which also happens to be called
-    <varname>perl</varname>.  However, it looks a bit silly, so there
-    is a shorter syntax.  The <literal>inherit</literal> keyword
-    causes the specified attributes to be bound to whatever variables
-    with the same name happen to be in scope.</para>
-
-  </listitem>
-
-</orderedlist>
-
-</para>
-
-</section>
diff --git a/doc/manual/expressions/generic-builder.xml b/doc/manual/expressions/generic-builder.xml
deleted file mode 100644
index a6f258abc..000000000
--- a/doc/manual/expressions/generic-builder.xml
+++ /dev/null
@@ -1,98 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='sec-generic-builder'>
-
-<title>Generic Builder Syntax</title>
-
-<para>Recall from <xref linkend='ex-hello-builder' /> that the builder
-looked something like this:
-
-<programlisting>
-PATH=$perl/bin:$PATH
-tar xvfz $src
-cd hello-*
-./configure --prefix=$out
-make
-make install</programlisting>
-
-The builders for almost all Unix packages look like this — set up some
-environment variables, unpack the sources, configure, build, and
-install.  For this reason the standard environment provides some Bash
-functions that automate the build process.  Here is what a builder
-using the generic build facilities looks like:</para>
-
-<programlisting>
-buildInputs="$perl" ①
-
-source $stdenv/setup ②
-
-genericBuild ③</programlisting>
-
-<para>Here is what each line means:
-
-<orderedlist>
-
-  <listitem>
-
-    <para>The <literal>buildInputs</literal> variable tells
-    <filename>setup</filename> to use the indicated packages as
-    <quote>inputs</quote>.  This means that if a package provides a
-    <filename>bin</filename> subdirectory, it's added to
-    <literal>PATH</literal>; if it has a <filename>include</filename>
-    subdirectory, it's added to GCC's header search path; and so
-    on. (This is implemented in a modular way:
-    <filename>setup</filename> tries to source the file
-    <filename><emphasis>pkg</emphasis>/nix-support/setup-hook</filename>
-    of all dependencies.  These “setup hooks” can then set up whatever
-    environment variables they want; for instance, the setup hook for
-    Perl sets the <literal>PERL5LIB</literal> environment variable to
-    contain the <filename>lib/site_perl</filename> directories of all
-    inputs.)
-    </para>
-
-  </listitem>
-
-  <listitem arearefs='ex-hello-builder2-co-2'>
-
-    <para>The function <function>genericBuild</function> is defined in
-    the file <literal>$stdenv/setup</literal>.</para>
-
-  </listitem>
-
-  <listitem arearefs='ex-hello-builder2-co-3'>
-
-    <para>The final step calls the shell function
-    <function>genericBuild</function>, which performs the steps that
-    were done explicitly in <xref linkend='ex-hello-builder' />.  The
-    generic builder is smart enough to figure out whether to unpack
-    the sources using <command>gzip</command>,
-    <command>bzip2</command>, etc.  It can be customised in many ways;
-    see the Nixpkgs manual for details.</para>
-
-  </listitem>
-
-</orderedlist>
-
-</para>
-
-<para>Discerning readers will note that the
-<literal>buildInputs</literal> could just as well have been set in the Nix
-expression, like this:
-
-<programlisting>
-  buildInputs = [ perl ];</programlisting>
-
-The <varname>perl</varname> attribute can then be removed, and the
-builder becomes even shorter:
-
-<programlisting>
-source $stdenv/setup
-genericBuild</programlisting>
-
-In fact, <varname>mkDerivation</varname> provides a default builder
-that looks exactly like that, so it is actually possible to omit the
-builder for Hello entirely.</para>
-
-</section>
diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
deleted file mode 100644
index e1c589f61..000000000
--- a/doc/manual/expressions/language-constructs.xml
+++ /dev/null
@@ -1,408 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-constructs">
-
-<title>Language Constructs</title>
-
-<section><title>Recursive sets</title>
-
-<para>Recursive sets are just normal sets, but the attributes can
-refer to each other.  For example,
-
-<programlisting>
-rec {
-  x = y;
-  y = 123;
-}.x
-</programlisting>
-
-evaluates to <literal>123</literal>.  Note that without
-<literal>rec</literal> the binding <literal>x = y;</literal> would
-refer to the variable <varname>y</varname> in the surrounding scope,
-if one exists, and would be invalid if no such variable exists.  That
-is, in a normal (non-recursive) set, attributes are not added to the
-lexical scope; in a recursive set, they are.</para>
-
-<para>Recursive sets of course introduce the danger of infinite
-recursion.  For example, the expression
-
-<programlisting>
-rec {
-  x = y;
-  y = x;
-}.x</programlisting>
-
-will crash with an <literal>infinite recursion encountered</literal>
-error message.</para>
-
-</section>
-
-
-<section xml:id="sect-let-expressions"><title>Let-expressions</title>
-
-<para>A let-expression allows you to define local variables for an
-expression.  For instance,
-
-<programlisting>
-let
-  x = "foo";
-  y = "bar";
-in x + y</programlisting>
-
-evaluates to <literal>"foobar"</literal>.
-
-</para>
-
-</section>
-
-
-<section><title>Inheriting attributes</title>
-
-<para>When defining a set or in a let-expression it is often convenient to copy variables
-from the surrounding lexical scope (e.g., when you want to propagate
-attributes).  This can be shortened using the
-<literal>inherit</literal> keyword.  For instance,
-
-<programlisting>
-let x = 123; in
-{ inherit x;
-  y = 456;
-}</programlisting>
-
-is equivalent to
-
-<programlisting>
-let x = 123; in
-{ x = x;
-  y = 456;
-}</programlisting>
-
-and both evaluate to <literal>{ x = 123; y = 456; }</literal>. (Note that
-this works because <varname>x</varname> is added to the lexical scope
-by the <literal>let</literal> construct.)  It is also possible to
-inherit attributes from another set.  For instance, in this fragment
-from <filename>all-packages.nix</filename>,
-
-<programlisting>
-  graphviz = (import ../tools/graphics/graphviz) {
-    inherit fetchurl stdenv libpng libjpeg expat x11 yacc;
-    inherit (xlibs) libXaw;
-  };
-
-  xlibs = {
-    libX11 = ...;
-    libXaw = ...;
-    ...
-  }
-
-  libpng = ...;
-  libjpg = ...;
-  ...</programlisting>
-
-the set used in the function call to the function defined in
-<filename>../tools/graphics/graphviz</filename> inherits a number of
-variables from the surrounding scope (<varname>fetchurl</varname>
-... <varname>yacc</varname>), but also inherits
-<varname>libXaw</varname> (the X Athena Widgets) from the
-<varname>xlibs</varname> (X11 client-side libraries) set.</para>
-
-<para>
-Summarizing the fragment
-
-<programlisting>
-...
-inherit x y z;
-inherit (src-set) a b c;
-...</programlisting>
-
-is equivalent to
-
-<programlisting>
-...
-x = x; y = y; z = z;
-a = src-set.a; b = src-set.b; c = src-set.c;
-...</programlisting>
-
-when used while defining local variables in a let-expression or
-while defining a set.</para>
-
-</section>
-
-
-<section xml:id="ss-functions"><title>Functions</title>
-
-<para>Functions have the following form:
-
-<programlisting>
-<emphasis>pattern</emphasis>: <emphasis>body</emphasis></programlisting>
-
-The pattern specifies what the argument of the function must look
-like, and binds variables in the body to (parts of) the
-argument.  There are three kinds of patterns:</para>
-
-<itemizedlist>
-
-
-  <listitem><para>If a pattern is a single identifier, then the
-  function matches any argument.  Example:
-
-  <programlisting>
-let negate = x: !x;
-    concat = x: y: x + y;
-in if negate true then concat "foo" "bar" else ""</programlisting>
-
-  Note that <function>concat</function> is a function that takes one
-  argument and returns a function that takes another argument.  This
-  allows partial parameterisation (i.e., only filling some of the
-  arguments of a function); e.g.,
-
-  <programlisting>
-map (concat "foo") [ "bar" "bla" "abc" ]</programlisting>
-
-  evaluates to <literal>[ "foobar" "foobla"
-  "fooabc" ]</literal>.</para></listitem>
-
-
-  <listitem><para>A <emphasis>set pattern</emphasis> of the form
-  <literal>{ name1, name2, …, nameN }</literal> matches a set
-  containing the listed attributes, and binds the values of those
-  attributes to variables in the function body.  For example, the
-  function
-
-<programlisting>
-{ x, y, z }: z + y + x</programlisting>
-
-  can only be called with a set containing exactly the attributes
-  <varname>x</varname>, <varname>y</varname> and
-  <varname>z</varname>.  No other attributes are allowed.  If you want
-  to allow additional arguments, you can use an ellipsis
-  (<literal>...</literal>):
-
-<programlisting>
-{ x, y, z, ... }: z + y + x</programlisting>
-
-  This works on any set that contains at least the three named
-  attributes.</para>
-
-  <para>It is possible to provide <emphasis>default values</emphasis>
-  for attributes, in which case they are allowed to be missing.  A
-  default value is specified by writing
-  <literal><emphasis>name</emphasis> ?
-  <emphasis>e</emphasis></literal>, where
-  <emphasis>e</emphasis> is an arbitrary expression.  For example,
-
-<programlisting>
-{ x, y ? "foo", z ? "bar" }: z + y + x</programlisting>
-
-  specifies a function that only requires an attribute named
-  <varname>x</varname>, but optionally accepts <varname>y</varname>
-  and <varname>z</varname>.</para></listitem>
-
-
-  <listitem><para>An <literal>@</literal>-pattern provides a means of referring
-  to the whole value being matched:
-
-<programlisting> args@{ x, y, z, ... }: z + y + x + args.a</programlisting>
-
-but can also be written as:
-
-<programlisting> { x, y, z, ... } @ args: z + y + x + args.a</programlisting>
-
-  Here <varname>args</varname> is bound to the entire argument, which
-  is further matched against the pattern <literal>{ x, y, z,
-  ... }</literal>. <literal>@</literal>-pattern makes mainly sense with an 
-  ellipsis(<literal>...</literal>) as you can access attribute names as 
-  <literal>a</literal>, using <literal>args.a</literal>, which was given as an
-  additional attribute to the function.
-  </para>
-
-  <warning>
-   <para>
-    The <literal>args@</literal> expression is bound to the argument passed to the function which
-    means that attributes with defaults that aren't explicitly specified in the function call
-    won't cause an evaluation error, but won't exist in <literal>args</literal>.
-   </para>
-   <para>
-    For instance
-<programlisting>
-let
-  function = args@{ a ? 23, ... }: args;
-in
- function {}
-</programlisting>
-    will evaluate to an empty attribute set.
-   </para>
-  </warning></listitem>
-
-</itemizedlist>
-
-<para>Note that functions do not have names.  If you want to give them
-a name, you can bind them to an attribute, e.g.,
-
-<programlisting>
-let concat = { x, y }: x + y;
-in concat { x = "foo"; y = "bar"; }</programlisting>
-
-</para>
-
-</section>
-
-
-<section><title>Conditionals</title>
-
-<para>Conditionals look like this:
-
-<programlisting>
-if <emphasis>e1</emphasis> then <emphasis>e2</emphasis> else <emphasis>e3</emphasis></programlisting>
-
-where <emphasis>e1</emphasis> is an expression that should
-evaluate to a Boolean value (<literal>true</literal> or
-<literal>false</literal>).</para>
-
-</section>
-
-
-<section><title>Assertions</title>
-
-<para>Assertions are generally used to check that certain requirements
-on or between features and dependencies hold.  They look like this:
-
-<programlisting>
-assert <emphasis>e1</emphasis>; <emphasis>e2</emphasis></programlisting>
-
-where <emphasis>e1</emphasis> is an expression that should
-evaluate to a Boolean value.  If it evaluates to
-<literal>true</literal>, <emphasis>e2</emphasis> is returned;
-otherwise expression evaluation is aborted and a backtrace is printed.</para>
-
-<para>Here is a Nix expression for the Subversion package that shows
-how assertions can be used:.</para>
-
-<programlisting>
-{ localServer ? false
-, httpServer ? false
-, sslSupport ? false
-, pythonBindings ? false
-, javaSwigBindings ? false
-, javahlBindings ? false
-, stdenv, fetchurl
-, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null
-}:
-
-assert localServer -> db4 != null; ①
-assert httpServer -> httpd != null &amp;&amp; httpd.expat == expat; ②
-assert sslSupport -> openssl != null &amp;&amp; (httpServer -> httpd.openssl == openssl); ③
-assert pythonBindings -> swig != null &amp;&amp; swig.pythonSupport;
-assert javaSwigBindings -> swig != null &amp;&amp; swig.javaSupport;
-assert javahlBindings -> j2sdk != null;
-
-stdenv.mkDerivation {
-  name = "subversion-1.1.1";
-  ...
-  openssl = if sslSupport then openssl else null; ④
-  ...
-}</programlisting>
-
-<para>The points of interest are:</para>
-
-<orderedlist>
-
-  <listitem>
-    <para>This assertion states that if Subversion is to have support
-    for local repositories, then Berkeley DB is needed.  So if the
-    Subversion function is called with the
-    <varname>localServer</varname> argument set to
-    <literal>true</literal> but the <varname>db4</varname> argument
-    set to <literal>null</literal>, then the evaluation fails.</para>
-  </listitem>
-
-  <listitem>
-    <para>This is a more subtle condition: if Subversion is built with
-    Apache (<literal>httpServer</literal>) support, then the Expat
-    library (an XML library) used by Subversion should be same as the
-    one used by Apache.  This is because in this configuration
-    Subversion code ends up being linked with Apache code, and if the
-    Expat libraries do not match, a build- or runtime link error or
-    incompatibility might occur.</para>
-  </listitem>
-
-  <listitem>
-    <para>This assertion says that in order for Subversion to have SSL
-    support (so that it can access <literal>https</literal> URLs), an
-    OpenSSL library must be passed.  Additionally, it says that
-    <emphasis>if</emphasis> Apache support is enabled, then Apache's
-    OpenSSL should match Subversion's.  (Note that if Apache support
-    is not enabled, we don't care about Apache's OpenSSL.)</para>
-  </listitem>
-
-  <listitem>
-    <para>The conditional here is not really related to assertions,
-    but is worth pointing out: it ensures that if SSL support is
-    disabled, then the Subversion derivation is not dependent on
-    OpenSSL, even if a non-<literal>null</literal> value was passed.
-    This prevents an unnecessary rebuild of Subversion if OpenSSL
-    changes.</para>
-  </listitem>
-
-</orderedlist>
-
-</section>
-
-
-
-<section><title>With-expressions</title>
-
-<para>A <emphasis>with-expression</emphasis>,
-
-<programlisting>
-with <emphasis>e1</emphasis>; <emphasis>e2</emphasis></programlisting>
-
-introduces the set <emphasis>e1</emphasis> into the lexical
-scope of the expression <emphasis>e2</emphasis>.  For instance,
-
-<programlisting>
-let as = { x = "foo"; y = "bar"; };
-in with as; x + y</programlisting>
-
-evaluates to <literal>"foobar"</literal> since the
-<literal>with</literal> adds the <varname>x</varname> and
-<varname>y</varname> attributes of <varname>as</varname> to the
-lexical scope in the expression <literal>x + y</literal>.  The most
-common use of <literal>with</literal> is in conjunction with the
-<function>import</function> function.  E.g.,
-
-<programlisting>
-with (import ./definitions.nix); ...</programlisting>
-
-makes all attributes defined in the file
-<filename>definitions.nix</filename> available as if they were defined
-locally in a <literal>let</literal>-expression.</para>
-
-<para>The bindings introduced by <literal>with</literal> do not shadow bindings
-introduced by other means, e.g.
-
-<programlisting>
-let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...</programlisting>
-
-establishes the same scope as
-
-<programlisting>
-let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...</programlisting>
-
-</para>
-
-</section>
-
-
-<section><title>Comments</title>
-
-<para>Comments can be single-line, started with a <literal>#</literal>
-character, or inline/multi-line, enclosed within <literal>/*
-... */</literal>.</para>
-
-</section>
-
-
-</section>
diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml
deleted file mode 100644
index 7f69bfcef..000000000
--- a/doc/manual/expressions/language-operators.xml
+++ /dev/null
@@ -1,222 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-language-operators">
-
-<title>Operators</title>
-
-<para><xref linkend='table-operators' /> lists the operators in the
-Nix expression language, in order of precedence (from strongest to
-weakest binding).</para>
-
-<table xml:id='table-operators'>
-  <title>Operators</title>
-  <tgroup cols='3'>
-    <thead>
-      <row>
-        <entry>Name</entry>
-        <entry>Syntax</entry>
-        <entry>Associativity</entry>
-        <entry>Description</entry>
-        <entry>Precedence</entry>
-      </row>
-    </thead>
-    <tbody>
-      <row>
-        <entry>Select</entry>
-        <entry><emphasis>e</emphasis> <literal>.</literal>
-        <emphasis>attrpath</emphasis>
-        [ <literal>or</literal> <emphasis>def</emphasis> ]
-        </entry>
-        <entry>none</entry>
-        <entry>Select attribute denoted by the attribute path
-        <emphasis>attrpath</emphasis> from set
-        <emphasis>e</emphasis>.  (An attribute path is a
-        dot-separated list of attribute names.)  If the attribute
-        doesn’t exist, return <emphasis>def</emphasis> if
-        provided, otherwise abort evaluation.</entry>
-        <entry>1</entry>
-      </row>
-      <row>
-        <entry>Application</entry>
-        <entry><emphasis>e1</emphasis> <emphasis>e2</emphasis></entry>
-        <entry>left</entry>
-        <entry>Call function <emphasis>e1</emphasis> with
-        argument <emphasis>e2</emphasis>.</entry>
-        <entry>2</entry>
-      </row>
-      <row>
-        <entry>Arithmetic Negation</entry>
-        <entry><literal>-</literal> <emphasis>e</emphasis></entry>
-        <entry>none</entry>
-        <entry>Arithmetic negation.</entry>
-        <entry>3</entry>
-      </row>
-      <row>
-        <entry>Has Attribute</entry>
-        <entry><emphasis>e</emphasis> <literal>?</literal>
-        <emphasis>attrpath</emphasis></entry>
-        <entry>none</entry>
-        <entry>Test whether set <emphasis>e</emphasis> contains
-        the attribute denoted by <emphasis>attrpath</emphasis>;
-        return <literal>true</literal> or
-        <literal>false</literal>.</entry>
-        <entry>4</entry>
-      </row>
-      <row>
-        <entry>List Concatenation</entry>
-        <entry><emphasis>e1</emphasis> <literal>++</literal> <emphasis>e2</emphasis></entry>
-        <entry>right</entry>
-        <entry>List concatenation.</entry>
-        <entry>5</entry>
-      </row>
-      <row>
-        <entry>Multiplication</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>*</literal> <emphasis>e2</emphasis>,
-        </entry>
-        <entry>left</entry>
-        <entry>Arithmetic multiplication.</entry>
-        <entry>6</entry>
-      </row>
-      <row>
-        <entry>Division</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>/</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>left</entry>
-        <entry>Arithmetic division.</entry>
-        <entry>6</entry>
-      </row>
-      <row>
-        <entry>Addition</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>+</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>left</entry>
-        <entry>Arithmetic addition.</entry>
-        <entry>7</entry>
-      </row>
-      <row>
-        <entry>Subtraction</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>-</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>left</entry>
-        <entry>Arithmetic subtraction.</entry>
-        <entry>7</entry>
-      </row>
-      <row>
-        <entry>String Concatenation</entry>
-        <entry>
-          <emphasis>string1</emphasis> <literal>+</literal> <emphasis>string2</emphasis>
-        </entry>
-        <entry>left</entry>
-        <entry>String concatenation.</entry>
-        <entry>7</entry>
-      </row>
-      <row>
-        <entry>Not</entry>
-        <entry><literal>!</literal> <emphasis>e</emphasis></entry>
-        <entry>none</entry>
-        <entry>Boolean negation.</entry>
-        <entry>8</entry>
-      </row>
-      <row>
-        <entry>Update</entry>
-        <entry><emphasis>e1</emphasis> <literal>//</literal>
-        <emphasis>e2</emphasis></entry>
-        <entry>right</entry>
-        <entry>Return a set consisting of the attributes in
-        <emphasis>e1</emphasis> and
-        <emphasis>e2</emphasis> (with the latter taking
-        precedence over the former in case of equally named
-        attributes).</entry>
-        <entry>9</entry>
-      </row>
-      <row>
-        <entry>Less Than</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>&lt;</literal> <emphasis>e2</emphasis>,
-        </entry>
-        <entry>none</entry>
-        <entry>Arithmetic comparison.</entry>
-        <entry>10</entry>
-      </row>
-      <row>
-        <entry>Less Than or Equal To</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>&lt;=</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>none</entry>
-        <entry>Arithmetic comparison.</entry>
-        <entry>10</entry>
-      </row>
-      <row>
-        <entry>Greater Than</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>&gt;</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>none</entry>
-        <entry>Arithmetic comparison.</entry>
-        <entry>10</entry>
-      </row>
-      <row>
-        <entry>Greater Than or Equal To</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>&gt;=</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>none</entry>
-        <entry>Arithmetic comparison.</entry>
-        <entry>10</entry>
-      </row>
-      <row>
-        <entry>Equality</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>==</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>none</entry>
-        <entry>Equality.</entry>
-        <entry>11</entry>
-      </row>
-      <row>
-        <entry>Inequality</entry>
-        <entry>
-          <emphasis>e1</emphasis> <literal>!=</literal> <emphasis>e2</emphasis>
-        </entry>
-        <entry>none</entry>
-        <entry>Inequality.</entry>
-        <entry>11</entry>
-      </row>
-      <row>
-        <entry>Logical AND</entry>
-        <entry><emphasis>e1</emphasis> <literal>&amp;&amp;</literal>
-        <emphasis>e2</emphasis></entry>
-        <entry>left</entry>
-        <entry>Logical AND.</entry>
-        <entry>12</entry>
-      </row>
-      <row>
-        <entry>Logical OR</entry>
-        <entry><emphasis>e1</emphasis> <literal>||</literal>
-        <emphasis>e2</emphasis></entry>
-        <entry>left</entry>
-        <entry>Logical OR.</entry>
-        <entry>13</entry>
-      </row>
-      <row>
-        <entry>Logical Implication</entry>
-        <entry><emphasis>e1</emphasis> <literal>-></literal>
-        <emphasis>e2</emphasis></entry>
-        <entry>none</entry>
-        <entry>Logical implication (equivalent to
-        <literal>!<emphasis>e1</emphasis> ||
-        <emphasis>e2</emphasis></literal>).</entry>
-        <entry>14</entry>
-      </row>
-    </tbody>
-  </tgroup>
-</table>
-
-</section>
diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml
deleted file mode 100644
index 5520a4938..000000000
--- a/doc/manual/expressions/language-values.xml
+++ /dev/null
@@ -1,312 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='ssec-values'>
-
-<title>Values</title>
-
-
-<section><title>Simple Values</title>
-
-<para>Nix has the following basic data types:
-
-<itemizedlist>
-
-  <listitem>
-
-    <para><emphasis>Strings</emphasis> can be written in three
-    ways.</para>
-
-    <para>The most common way is to enclose the string between double
-    quotes, e.g., <literal>"foo bar"</literal>.  Strings can span
-    multiple lines.  The special characters <literal>"</literal> and
-    <literal>\</literal> and the character sequence
-    <literal>${</literal> must be escaped by prefixing them with a
-    backslash (<literal>\</literal>).  Newlines, carriage returns and
-    tabs can be written as <literal>\n</literal>,
-    <literal>\r</literal> and <literal>\t</literal>,
-    respectively.</para>
-
-    <para>You can include the result of an expression into a string by
-    enclosing it in
-    <literal>${<emphasis>...</emphasis>}</literal>, a feature
-    known as <emphasis>antiquotation</emphasis>.  The enclosed
-    expression must evaluate to something that can be coerced into a
-    string (meaning that it must be a string, a path, or a
-    derivation).  For instance, rather than writing
-
-<programlisting>
-"--with-freetype2-library=" + freetype + "/lib"</programlisting>
-
-    (where <varname>freetype</varname> is a derivation), you can
-    instead write the more natural
-
-<programlisting>
-"--with-freetype2-library=${freetype}/lib"</programlisting>
-
-    The latter is automatically translated to the former.  A more
-    complicated example (from the Nix expression for <link
-    xlink:href='http://www.trolltech.com/products/qt'>Qt</link>):
-
-<programlisting>
-configureFlags = "
-  -system-zlib -system-libpng -system-libjpeg
-  ${if openglSupport then "-dlopen-opengl
-    -L${mesa}/lib -I${mesa}/include
-    -L${libXmu}/lib -I${libXmu}/include" else ""}
-  ${if threadSupport then "-thread" else "-no-thread"}
-";</programlisting>
-
-    Note that Nix expressions and strings can be arbitrarily nested;
-    in this case the outer string contains various antiquotations that
-    themselves contain strings (e.g., <literal>"-thread"</literal>),
-    some of which in turn contain expressions (e.g.,
-    <literal>${mesa}</literal>).</para>
-
-    <para>The second way to write string literals is as an
-    <emphasis>indented string</emphasis>, which is enclosed between
-    pairs of <emphasis>double single-quotes</emphasis>, like so:
-
-<programlisting>
-''
-  This is the first line.
-  This is the second line.
-    This is the third line.
-''</programlisting>
-
-    This kind of string literal intelligently strips indentation from
-    the start of each line.  To be precise, it strips from each line a
-    number of spaces equal to the minimal indentation of the string as
-    a whole (disregarding the indentation of empty lines).  For
-    instance, the first and second line are indented two space, while
-    the third line is indented four spaces.  Thus, two spaces are
-    stripped from each line, so the resulting string is
-
-<programlisting>
-"This is the first line.\nThis is the second line.\n  This is the third line.\n"</programlisting>
-
-    </para>
-
-    <para>Note that the whitespace and newline following the opening
-    <literal>''</literal> is ignored if there is no non-whitespace
-    text on the initial line.</para>
-
-    <para>Antiquotation
-    (<literal>${<emphasis>expr</emphasis>}</literal>) is
-    supported in indented strings.</para>
-
-    <para>Since <literal>${</literal> and <literal>''</literal> have
-    special meaning in indented strings, you need a way to quote them.
-    <literal>$</literal> can be escaped by prefixing it with
-    <literal>''</literal> (that is, two single quotes), i.e.,
-    <literal>''$</literal>. <literal>''</literal> can be escaped by
-    prefixing it with <literal>'</literal>, i.e.,
-    <literal>'''</literal>. <literal>$</literal> removes any special meaning
-    from the following <literal>$</literal>. Linefeed, carriage-return and tab
-    characters can be written as <literal>''\n</literal>,
-    <literal>''\r</literal>, <literal>''\t</literal>, and <literal>''\</literal>
-    escapes any other character.
-
-    </para>
-
-    <para>Indented strings are primarily useful in that they allow
-    multi-line string literals to follow the indentation of the
-    enclosing Nix expression, and that less escaping is typically
-    necessary for strings representing languages such as shell scripts
-    and configuration files because <literal>''</literal> is much less
-    common than <literal>"</literal>.  Example:
-
-<programlisting>
-stdenv.mkDerivation {
-  <emphasis>...</emphasis>
-  postInstall =
-    ''
-      mkdir $out/bin $out/etc
-      cp foo $out/bin
-      echo "Hello World" > $out/etc/foo.conf
-      ${if enableBar then "cp bar $out/bin" else ""}
-    '';
-  <emphasis>...</emphasis>
-}
-</programlisting>
-
-    </para>
-
-    <para>Finally, as a convenience, <emphasis>URIs</emphasis> as
-    defined in appendix B of <link
-    xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC 2396</link>
-    can be written <emphasis>as is</emphasis>, without quotes.  For
-    instance, the string
-    <literal>"http://example.org/foo.tar.bz2"</literal>
-    can also be written as
-    <literal>http://example.org/foo.tar.bz2</literal>.</para>
-
-  </listitem>
-
-  <listitem><para>Numbers, which can be <emphasis>integers</emphasis> (like
-  <literal>123</literal>) or <emphasis>floating point</emphasis> (like
-  <literal>123.43</literal> or <literal>.27e13</literal>).</para>
-
-  <para>Numbers are type-compatible: pure integer operations will always
-  return integers, whereas any operation involving at least one floating point
-  number will have a floating point number as a result.</para></listitem>
-
-  <listitem><para><emphasis>Paths</emphasis>, e.g.,
-  <filename>/bin/sh</filename> or <filename>./builder.sh</filename>.
-  A path must contain at least one slash to be recognised as such. For
-  instance, <filename>builder.sh</filename> is not a path: it's parsed
-  as an expression that selects the attribute <varname>sh</varname>
-  from the variable <varname>builder</varname>. If the file name is
-  relative, i.e., if it does not begin with a slash, it is made
-  absolute at parse time relative to the directory of the Nix
-  expression that contained it.  For instance, if a Nix expression in
-  <filename>/foo/bar/bla.nix</filename> refers to
-  <filename>../xyzzy/fnord.nix</filename>, the absolute path is
-  <filename>/foo/xyzzy/fnord.nix</filename>.</para>
-
-  <para>If the first component of a path is a <literal>~</literal>,
-  it is interpreted as if the rest of the path were relative to the
-  user's home directory. e.g. <filename>~/foo</filename> would be
-  equivalent to <filename>/home/edolstra/foo</filename> for a user
-  whose home directory is <filename>/home/edolstra</filename>.
-  </para>
-
-  <para>Paths can also be specified between angle brackets, e.g.
-  <literal>&lt;nixpkgs&gt;</literal>. This means that the directories
-  listed in the environment variable
-  <literal linkend="env-NIX_PATH">NIX_PATH</literal> will be searched
-  for the given file or directory name.
-  </para>
-
-  </listitem>
-
-  <listitem><para><emphasis>Booleans</emphasis> with values
-  <literal>true</literal> and
-  <literal>false</literal>.</para></listitem>
-
-  <listitem><para>The null value, denoted as
-  <literal>null</literal>.</para></listitem>
-
-</itemizedlist>
-
-</para>
-
-</section>
-
-
-<section><title>Lists</title>
-
-<para>Lists are formed by enclosing a whitespace-separated list of
-values between square brackets.  For example,
-
-<programlisting>
-[ 123 ./foo.nix "abc" (f { x = y; }) ]</programlisting>
-
-defines a list of four elements, the last being the result of a call
-to the function <varname>f</varname>.  Note that function calls have
-to be enclosed in parentheses.  If they had been omitted, e.g.,
-
-<programlisting>
-[ 123 ./foo.nix "abc" f { x = y; } ]</programlisting>
-
-the result would be a list of five elements, the fourth one being a
-function and the fifth being a set.</para>
-
-<para>Note that lists are only lazy in values, and they are strict in length.
-</para>
-
-</section>
-
-
-<section><title>Sets</title>
-
-<para>Sets are really the core of the language, since ultimately the
-Nix language is all about creating derivations, which are really just
-sets of attributes to be passed to build scripts.</para>
-
-<para>Sets are just a list of name/value pairs (called
-<emphasis>attributes</emphasis>) enclosed in curly brackets, where
-each value is an arbitrary expression terminated by a semicolon.  For
-example:
-
-<programlisting>
-{ x = 123;
-  text = "Hello";
-  y = f { bla = 456; };
-}</programlisting>
-
-This defines a set with attributes named <varname>x</varname>,
-<varname>text</varname>, <varname>y</varname>.  The order of the
-attributes is irrelevant.  An attribute name may only occur
-once.</para>
-
-<para>Attributes can be selected from a set using the
-<literal>.</literal> operator.  For instance,
-
-<programlisting>
-{ a = "Foo"; b = "Bar"; }.a</programlisting>
-
-evaluates to <literal>"Foo"</literal>.  It is possible to provide a
-default value in an attribute selection using the
-<literal>or</literal> keyword.  For example,
-
-<programlisting>
-{ a = "Foo"; b = "Bar"; }.c or "Xyzzy"</programlisting>
-
-will evaluate to <literal>"Xyzzy"</literal> because there is no
-<varname>c</varname> attribute in the set.</para>
-
-<para>You can use arbitrary double-quoted strings as attribute
-names:
-
-<programlisting>
-{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}"
-</programlisting>
-
-This will evaluate to <literal>123</literal> (Assuming
-<literal>bar</literal> is antiquotable). In the case where an
-attribute name is just a single antiquotation, the quotes can be
-dropped:
-
-<programlisting>
-{ foo = 123; }.${bar} or 456 </programlisting>
-
-This will evaluate to <literal>123</literal> if
-<literal>bar</literal> evaluates to <literal>"foo"</literal> when
-coerced to a string and <literal>456</literal> otherwise (again
-assuming <literal>bar</literal> is antiquotable).</para>
-
-<para>In the special case where an attribute name inside of a set declaration
-evaluates to <literal>null</literal> (which is normally an error, as
-<literal>null</literal> is not antiquotable), that attribute is simply not
-added to the set:
-
-<programlisting>
-{ ${if foo then "bar" else null} = true; }</programlisting>
-
-This will evaluate to <literal>{}</literal> if <literal>foo</literal>
-evaluates to <literal>false</literal>.</para>
-
-<para>A set that has a <literal>__functor</literal> attribute whose value
-is callable (i.e. is itself a function or a set with a
-<literal>__functor</literal> attribute whose value is callable) can be
-applied as if it were a function, with the set itself passed in first
-, e.g.,
-
-<programlisting>
-let add = { __functor = self: x: x + self.x; };
-    inc = add // { x = 1; };
-in inc 1
-</programlisting>
-
-evaluates to <literal>2</literal>. This can be used to attach metadata to a
-function without the caller needing to treat it specially, or to implement
-a form of object-oriented programming, for example.
-
-</para>
-
-</section>
-
-
-</section>
diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml
deleted file mode 100644
index f82223df9..000000000
--- a/doc/manual/expressions/simple-building-testing.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='sec-building-simple'>
-
-<title>Building and Testing</title>
-
-<para>You can now try to build Hello.  Of course, you could do
-<literal>nix-env -i hello</literal>, but you may not want to install a
-possibly broken package just yet.  The best way to test the package is by
-using the command <command linkend="sec-nix-build">nix-build</command>,
-which builds a Nix expression and creates a symlink named
-<filename>result</filename> in the current directory:
-
-<screen>
-$ nix-build -A hello
-building path `/nix/store/632d2b22514d...-hello-2.1.1'
-hello-2.1.1/
-hello-2.1.1/intl/
-hello-2.1.1/intl/ChangeLog
-<emphasis>...</emphasis>
-
-$ ls -l result
-lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1
-
-$ ./result/bin/hello
-Hello, world!</screen>
-
-The <link linkend='opt-attr'><option>-A</option></link> option selects
-the <literal>hello</literal> attribute.  This is faster than using the
-symbolic package name specified by the <literal>name</literal>
-attribute (which also happens to be <literal>hello</literal>) and is
-unambiguous (there can be multiple packages with the symbolic name
-<literal>hello</literal>, but there can be only one attribute in a set
-named <literal>hello</literal>).</para>
-
-<para><command>nix-build</command> registers the
-<filename>./result</filename> symlink as a garbage collection root, so
-unless and until you delete the <filename>./result</filename> symlink,
-the output of the build will be safely kept on your system.  You can
-use <command>nix-build</command>’s <option
-linkend='opt-out-link'>-o</option> switch to give the symlink another
-name.</para>
-
-<para>Nix has transactional semantics.  Once a build finishes
-successfully, Nix makes a note of this in its database: it registers
-that the path denoted by <literal>out</literal> is now
-<quote>valid</quote>.  If you try to build the derivation again, Nix
-will see that the path is already valid and finish immediately.  If a
-build fails, either because it returns a non-zero exit code, because
-Nix or the builder are killed, or because the machine crashes, then
-the output paths will not be registered as valid.  If you try to build
-the derivation again, Nix will remove the output paths if they exist
-(e.g., because the builder died half-way through <literal>make
-install</literal>) and try again.  Note that there is no
-<quote>negative caching</quote>: Nix doesn't remember that a build
-failed, and so a failed build can always be repeated.  This is because
-Nix cannot distinguish between permanent failures (e.g., a compiler
-error due to a syntax error in the source) and transient failures
-(e.g., a disk full condition).</para>
-
-<para>Nix also performs locking.  If you run multiple Nix builds
-simultaneously, and they try to build the same derivation, the first
-Nix instance that gets there will perform the build, while the others
-block (or perform other derivations if available) until the build
-finishes:
-
-<screen>
-$ nix-build -A hello
-waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</screen>
-
-So it is always safe to run multiple instances of Nix in parallel
-(which isn’t the case with, say, <command>make</command>).</para>
-
-</section>
diff --git a/doc/manual/expressions/simple-expression.xml b/doc/manual/expressions/simple-expression.xml
deleted file mode 100644
index ad97220a8..000000000
--- a/doc/manual/expressions/simple-expression.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-simple-expression">
-
-<title>A Simple Nix Expression</title>
-
-<para>This section shows how to add and test the <link
-xlink:href='http://www.gnu.org/software/hello/hello.html'>GNU Hello
-package</link> to the Nix Packages collection.  Hello is a program
-that prints out the text <quote>Hello, world!</quote>.</para>
-
-<para>To add a package to the Nix Packages collection, you generally
-need to do three things:
-
-<orderedlist>
-
-  <listitem><para>Write a Nix expression for the package.  This is a
-  file that describes all the inputs involved in building the package,
-  such as dependencies, sources, and so on.</para></listitem>
-
-  <listitem><para>Write a <emphasis>builder</emphasis>.  This is a
-  shell script that builds the package from the inputs. (In fact, it
-  can be written in any language, but typically it's a
-  <command>bash</command> shell script.)</para></listitem>
-
-  <listitem><para>Add the package to the file
-  <filename>pkgs/top-level/all-packages.nix</filename>.  The Nix
-  expression written in the first step is a
-  <emphasis>function</emphasis>; it requires other packages in order
-  to build it.  In this step you put it all together, i.e., you call
-  the function with the right arguments to build the actual
-  package.</para></listitem>
-
-</orderedlist>
-
-</para>
-
-<xi:include href="expression-syntax.xml" />
-<xi:include href="build-script.xml" />
-<xi:include href="arguments-variables.xml" />
-<xi:include href="simple-building-testing.xml" />
-<xi:include href="generic-builder.xml" />
-
-</chapter>
diff --git a/doc/manual/expressions/writing-nix-expressions.xml b/doc/manual/expressions/writing-nix-expressions.xml
deleted file mode 100644
index 6646dddf0..000000000
--- a/doc/manual/expressions/writing-nix-expressions.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<part xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='chap-writing-nix-expressions'>
-
-<title>Writing Nix Expressions</title>
-
-<partintro>
-<para>This chapter shows you how to write Nix expressions, which 
-instruct Nix how to build packages.  It starts with a
-simple example (a Nix expression for GNU Hello), and then moves
-on to a more in-depth look at the Nix expression language.</para>
-
-<note><para>This chapter is mostly about the Nix expression language.
-For more extensive information on adding packages to the Nix Packages
-collection (such as functions in the standard environment and coding
-conventions), please consult <link
-xlink:href="http://nixos.org/nixpkgs/manual/">its
-manual</link>.</para></note>
-</partintro>
-
-<xi:include href="simple-expression.xml" />
-<xi:include href="expression-language.xml" />
-
-</part>
diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml
deleted file mode 100644
index e3162ed8d..000000000
--- a/doc/manual/glossary/glossary.xml
+++ /dev/null
@@ -1,199 +0,0 @@
-<appendix xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xml:id="part-glossary">
-
-<title>Glossary</title>
-
-
-<glosslist>
-
-
-<glossentry xml:id="gloss-derivation"><glossterm>derivation</glossterm>
-
-  <glossdef><para>A description of a build action.  The result of a
-  derivation is a store object.  Derivations are typically specified
-  in Nix expressions using the <link
-  linkend="ssec-derivation"><function>derivation</function>
-  primitive</link>.  These are translated into low-level
-  <emphasis>store derivations</emphasis> (implicitly by
-  <command>nix-env</command> and <command>nix-build</command>, or
-  explicitly by <command>nix-instantiate</command>).</para></glossdef>
-
-</glossentry>
-
-
-<glossentry><glossterm>store</glossterm>
-
-  <glossdef><para>The location in the file system where store objects
-  live.  Typically <filename>/nix/store</filename>.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry><glossterm>store path</glossterm>
-
-  <glossdef><para>The location in the file system of a store object,
-  i.e., an immediate child of the Nix store
-  directory.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry><glossterm>store object</glossterm>
-
-  <glossdef><para>A file that is an immediate child of the Nix store
-  directory.  These can be regular files, but also entire directory
-  trees.  Store objects can be sources (objects copied from outside of
-  the store), derivation outputs (objects produced by running a build
-  action), or derivations (files describing a build
-  action).</para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-substitute"><glossterm>substitute</glossterm>
-
-  <glossdef><para>A substitute is a command invocation stored in the
-  Nix database that describes how to build a store object, bypassing
-  the normal build mechanism (i.e., derivations).  Typically, the
-  substitute builds the store object by downloading a pre-built
-  version of the store object from some server.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry><glossterm>purity</glossterm>
-
-  <glossdef><para>The assumption that equal Nix derivations when run
-  always produce the same output.  This cannot be guaranteed in
-  general (e.g., a builder can rely on external inputs such as the
-  network or the system time) but the Nix model assumes
-  it.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry><glossterm>Nix expression</glossterm>
-
-  <glossdef><para>A high-level description of software packages and
-  compositions thereof.  Deploying software using Nix entails writing
-  Nix expressions for your packages.  Nix expressions are translated
-  to derivations that are stored in the Nix store.  These derivations
-  can then be built.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-reference"><glossterm>reference</glossterm>
-
-  <glossdef>
-    <para>A store path <varname>P</varname> is said to have a
-    reference to a store path <varname>Q</varname> if the store object
-    at <varname>P</varname> contains the path <varname>Q</varname>
-    somewhere. The <emphasis>references</emphasis> of a store path are
-    the set of store paths to which it has a reference.
-    </para>
-    <para>A derivation can reference other derivations and sources
-    (but not output paths), whereas an output path only references other
-    output paths.
-    </para>
-  </glossdef>
-
-</glossentry>
-
-<glossentry xml:id="gloss-reachable"><glossterm>reachable</glossterm>
-
-  <glossdef><para>A store path <varname>Q</varname> is reachable from
-  another store path <varname>P</varname> if <varname>Q</varname> is in the
-  <link linkend="gloss-closure">closure</link> of the
-  <link linkend="gloss-reference">references</link> relation.
-  </para></glossdef>
-</glossentry>
-
-<glossentry xml:id="gloss-closure"><glossterm>closure</glossterm>
-
-  <glossdef><para>The closure of a store path is the set of store
-  paths that are directly or indirectly “reachable” from that store
-  path; that is, it’s the closure of the path under the <link
-  linkend="gloss-reference">references</link> relation. For a package, the
-  closure of its derivation is equivalent to the build-time
-  dependencies, while the closure of its output path is equivalent to its
-  runtime dependencies. For correct deployment it is necessary to deploy whole
-  closures, since otherwise at runtime files could be missing. The command
-  <command>nix-store -qR</command> prints out closures of store paths.
-  </para>
-  <para>As an example, if the store object at path <varname>P</varname> contains
-  a reference to path <varname>Q</varname>, then <varname>Q</varname> is
-  in the closure of <varname>P</varname>. Further, if <varname>Q</varname>
-  references <varname>R</varname> then <varname>R</varname> is also in
-  the closure of <varname>P</varname>.
-  </para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-output-path"><glossterm>output path</glossterm>
-
-  <glossdef><para>A store path produced by a derivation.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-deriver"><glossterm>deriver</glossterm>
-
-  <glossdef><para>The deriver of an <link
-  linkend="gloss-output-path">output path</link> is the store
-  derivation that built it.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-validity"><glossterm>validity</glossterm>
-
-  <glossdef><para>A store path is considered
-  <emphasis>valid</emphasis> if it exists in the file system, is
-  listed in the Nix database as being valid, and if all paths in its
-  closure are also valid.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-user-env"><glossterm>user environment</glossterm>
-
-  <glossdef><para>An automatically generated store object that
-  consists of a set of symlinks to “active” applications, i.e., other
-  store paths.  These are generated automatically by <link
-  linkend="sec-nix-env"><command>nix-env</command></link>.  See <xref
-  linkend="sec-profiles" />.</para>
-
-  </glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-profile"><glossterm>profile</glossterm>
-
-  <glossdef><para>A symlink to the current <link
-  linkend="gloss-user-env">user environment</link> of a user, e.g.,
-  <filename>/nix/var/nix/profiles/default</filename>.</para></glossdef>
-
-</glossentry>
-
-
-<glossentry xml:id="gloss-nar"><glossterm>NAR</glossterm>
-
-  <glossdef><para>A <emphasis>N</emphasis>ix
-  <emphasis>AR</emphasis>chive.  This is a serialisation of a path in
-  the Nix store.  It can contain regular files, directories and
-  symbolic links.  NARs are generated and unpacked using
-  <command>nix-store --dump</command> and <command>nix-store
-  --restore</command>.</para></glossdef>
-
-</glossentry>
-
-
-
-</glosslist>
-
-
-</appendix>
diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml
deleted file mode 100644
index 5d9f08d38..000000000
--- a/doc/manual/hacking.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<appendix xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xml:id="chap-hacking">
-
-<title>Hacking</title>
-
-<para>This section provides some notes on how to hack on Nix. To get
-the latest version of Nix from GitHub:
-<screen>
-$ git clone https://github.com/NixOS/nix.git
-$ cd nix
-</screen>
-</para>
-
-<para>To build Nix for the current operating system/architecture use
-
-<screen>
-$ nix-build
-</screen>
-
-or if you have a flakes-enabled nix:
-
-<screen>
-$ nix build
-</screen>
-
-This will build <literal>defaultPackage</literal> attribute defined in the <literal>flake.nix</literal> file.
-
-To build for other platforms add one of the following suffixes to it: aarch64-linux,
-i686-linux, x86_64-darwin, x86_64-linux.
-
-i.e.
-
-<screen>
-$ nix-build -A defaultPackage.x86_64-linux
-</screen>
-
-</para>
-
-<para>To build all dependencies and start a shell in which all
-environment variables are set up so that those dependencies can be
-found:
-<screen>
-$ nix-shell
-</screen>
-To build Nix itself in this shell:
-<screen>
-[nix-shell]$ ./bootstrap.sh
-[nix-shell]$ ./configure $configureFlags
-[nix-shell]$ make -j $NIX_BUILD_CORES
-</screen>
-To install it in <literal>$(pwd)/inst</literal> and test it:
-<screen>
-[nix-shell]$ make install
-[nix-shell]$ make installcheck
-[nix-shell]$ ./inst/bin/nix --version
-nix (Nix) 2.4
-</screen>
-
-If you have a flakes-enabled nix you can replace:
-
-<screen>
-$ nix-shell
-</screen>
-
-by:
-
-<screen>
-$ nix develop
-</screen>
-
-</para>
-
-</appendix>
diff --git a/doc/manual/installation/building-source.xml b/doc/manual/installation/building-source.xml
deleted file mode 100644
index 469aaebe9..000000000
--- a/doc/manual/installation/building-source.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-building-source">
-
-<title>Building Nix from Source</title>
-
-<para>After unpacking or checking out the Nix sources, issue the
-following commands:
-
-<screen>
-$ ./configure <emphasis>options...</emphasis>
-$ make
-$ make install</screen>
-
-Nix requires GNU Make so you may need to invoke
-<command>gmake</command> instead.</para>
-
-<para>When building from the Git repository, these should be preceded
-by the command:
-
-<screen>
-$ ./bootstrap.sh</screen>
-
-</para>
-
-<para>The installation path can be specified by passing the
-<option>--prefix=<emphasis>prefix</emphasis></option> to
-<command>configure</command>.  The default installation directory is
-<filename>/usr/local</filename>.  You can change this to any location
-you like.  You must have write permission to the
-<emphasis>prefix</emphasis> path.</para>
-
-<para>Nix keeps its <emphasis>store</emphasis> (the place where
-packages are stored) in <filename>/nix/store</filename> by default.
-This can be changed using
-<option>--with-store-dir=<emphasis>path</emphasis></option>.</para>
-
-<warning><para>It is best <emphasis>not</emphasis> to change the Nix
-store from its default, since doing so makes it impossible to use
-pre-built binaries from the standard Nixpkgs channels — that is, all
-packages will need to be built from source.</para></warning>
-
-<para>Nix keeps state (such as its database and log files) in
-<filename>/nix/var</filename> by default.  This can be changed using
-<option>--localstatedir=<emphasis>path</emphasis></option>.</para>
-
-</section>
diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
deleted file mode 100644
index cbce8559a..000000000
--- a/doc/manual/installation/env-variables.xml
+++ /dev/null
@@ -1,89 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-env-variables">
-
-<title>Environment Variables</title>
-
-<para>To use Nix, some environment variables should be set.  In
-particular, <literal>PATH</literal> should contain the directories
-<filename><emphasis>prefix</emphasis>/bin</filename> and
-<filename>~/.nix-profile/bin</filename>.  The first directory contains
-the Nix tools themselves, while <filename>~/.nix-profile</filename> is
-a symbolic link to the current <emphasis>user environment</emphasis>
-(an automatically generated package consisting of symlinks to
-installed packages).  The simplest way to set the required environment
-variables is to include the file
-<filename><emphasis>prefix</emphasis>/etc/profile.d/nix.sh</filename>
-in your <filename>~/.profile</filename> (or similar), like this:</para>
-
-<screen>
-source <emphasis>prefix</emphasis>/etc/profile.d/nix.sh</screen>
-
-<section xml:id="sec-nix-ssl-cert-file">
-
-<title><literal>NIX_SSL_CERT_FILE</literal></title>
-
-<para>If you need to specify a custom certificate bundle to account
-for an HTTPS-intercepting man in the middle proxy, you must specify
-the path to the certificate bundle in the environment variable
-<literal>NIX_SSL_CERT_FILE</literal>.</para>
-
-
-<para>If you don't specify a <literal>NIX_SSL_CERT_FILE</literal>
-manually, Nix will install and use its own certificate
-bundle.</para>
-
-<procedure>
-  <step><para>Set the environment variable and install Nix</para>
-    <screen>
-$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh &lt;(curl -L https://nixos.org/nix/install)
-</screen></step>
-
-  <step><para>In the shell profile and rc files (for example,
-  <filename>/etc/bashrc</filename>, <filename>/etc/zshrc</filename>),
-  add the following line:</para>
-<programlisting>
-export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-</programlisting>
-</step>
-</procedure>
-
-<note><para>You must not add the export and then do the install, as
-the Nix installer will detect the presense of Nix configuration, and
-abort.</para></note>
-
-<section xml:id="sec-nix-ssl-cert-file-with-nix-daemon-and-macos">
-<title><literal>NIX_SSL_CERT_FILE</literal> with macOS and the Nix daemon</title>
-
-<para>On macOS you must specify the environment variable for the Nix
-daemon service, then restart it:</para>
-
-<screen>
-$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt
-$ sudo launchctl kickstart -k system/org.nixos.nix-daemon
-</screen>
-</section>
-
-<section xml:id="sec-installer-proxy-settings">
-
-<title>Proxy Environment Variables</title>
-
-<para>The Nix installer has special handling for these proxy-related
-environment variables:
-<varname>http_proxy</varname>, <varname>https_proxy</varname>,
-<varname>ftp_proxy</varname>, <varname>no_proxy</varname>,
-<varname>HTTP_PROXY</varname>, <varname>HTTPS_PROXY</varname>,
-<varname>FTP_PROXY</varname>, <varname>NO_PROXY</varname>.
-</para>
-<para>If any of these variables are set when running the Nix installer,
-then the installer will create an override file at
-<filename>/etc/systemd/system/nix-daemon.service.d/override.conf</filename>
-so <command>nix-daemon</command> will use them.
-</para>
-</section>
-
-</section>
-</chapter>
diff --git a/doc/manual/installation/installation.xml b/doc/manual/installation/installation.xml
deleted file mode 100644
index 878959352..000000000
--- a/doc/manual/installation/installation.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<part xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="chap-installation">
-
-<title>Installation</title>
-
-<partintro>
-<para>This section describes how to install and configure Nix for first-time use.</para>
-</partintro>
-
-<xi:include href="supported-platforms.xml" />
-<xi:include href="installing-binary.xml" />
-<xi:include href="installing-source.xml" />
-<xi:include href="nix-security.xml" />
-<xi:include href="env-variables.xml" />
-
-<!-- TODO: should be updated
-<section><title>Upgrading Nix through Nix</title>
-
-<para>You can install the latest stable version of Nix through Nix
-itself by subscribing to the channel <link
-xlink:href="http://nixos.org/releases/nix/channels/nix-stable" />,
-or the latest unstable version by subscribing to the channel <link
-xlink:href="http://nixos.org/releases/nix/channels/nix-unstable" />.
-You can also do a <link linkend="sec-one-click">one-click
-installation</link> by clicking on the package links at <link
-xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
-
-</section>
--->
-
-</part>
diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
deleted file mode 100644
index 439198e6c..000000000
--- a/doc/manual/installation/installing-binary.xml
+++ /dev/null
@@ -1,469 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-installing-binary">
-
-<title>Installing a Binary Distribution</title>
-
-<para>
-  If you are using Linux or macOS versions up to 10.14 (Mojave), the
-  easiest way to install Nix is to run the following command:
-</para>
-
-<screen>
-  $ sh &lt;(curl -L https://nixos.org/nix/install)
-</screen>
-
-<para>
-  If you're using macOS 10.15 (Catalina) or newer, consult
-  <link linkend="sect-macos-installation">the macOS installation instructions</link>
-  before installing.
-</para>
-
-<para>
-  As of Nix 2.1.0, the Nix installer will always default to creating a
-  single-user installation, however opting in to the multi-user
-  installation is highly recommended.
-  <!-- TODO: this explains *neither* why the default version is
-  single-user, nor why we'd recommend multi-user over the default.
-  True prospective users don't have much basis for evaluating this.
-  What's it to me? Who should pick which? Why? What if I pick wrong?
-  -->
-</para>
-
-<section xml:id="sect-single-user-installation">
-  <title>Single User Installation</title>
-
-  <para>
-    To explicitly select a single-user installation on your system:
-
-    <screen>
-  sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
-</screen>
-  </para>
-
-<para>
-This will perform a single-user installation of Nix, meaning that
-<filename>/nix</filename> is owned by the invoking user.  You should
-run this under your usual user account, <emphasis>not</emphasis> as
-root.  The script will invoke <command>sudo</command> to create
-<filename>/nix</filename> if it doesn’t already exist.  If you don’t
-have <command>sudo</command>, you should manually create
-<filename>/nix</filename> first as root, e.g.:
-
-<screen>
-$ mkdir /nix
-$ chown alice /nix
-</screen>
-
-The install script will modify the first writable file from amongst
-<filename>.bash_profile</filename>, <filename>.bash_login</filename>
-and <filename>.profile</filename> to source
-<filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <literal>NIX_INSTALLER_NO_MODIFY_PROFILE</literal> environment
-variable before executing the install script to disable this
-behaviour.
-</para>
-
-
-<para>You can uninstall Nix simply by running:
-
-<screen>
-$ rm -rf /nix
-</screen>
-
-</para>
-</section>
-
-<section xml:id="sect-multi-user-installation">
-  <title>Multi User Installation</title>
-  <para>
-    The multi-user Nix installation creates system users, and a system
-    service for the Nix daemon.
-  </para>
-
-  <itemizedlist>
-    <title>Supported Systems</title>
-
-    <listitem>
-      <para>Linux running systemd, with SELinux disabled</para>
-    </listitem>
-    <listitem><para>macOS</para></listitem>
-  </itemizedlist>
-
-  <para>
-    You can instruct the installer to perform a multi-user
-    installation on your system:
-  </para>
-
-  <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
-
-  <para>
-    The multi-user installation of Nix will create build users between
-    the user IDs 30001 and 30032, and a group with the group ID 30000.
-
-    You should run this under your usual user account,
-    <emphasis>not</emphasis> as root. The script will invoke
-    <command>sudo</command> as needed.
-  </para>
-
-  <note><para>
-    If you need Nix to use a different group ID or user ID set, you
-    will have to download the tarball manually and <link
-    linkend="sect-nix-install-binary-tarball">edit the install
-    script</link>.
-  </para></note>
-
-  <para>
-    The installer will modify <filename>/etc/bashrc</filename>, and
-    <filename>/etc/zshrc</filename> if they exist. The installer will
-    first back up these files with a
-    <literal>.backup-before-nix</literal> extension. The installer
-    will also create <filename>/etc/profile.d/nix.sh</filename>.
-  </para>
-
-  <para>You can uninstall Nix with the following commands:
-
-<screen>
-sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
-
-# If you are on Linux with systemd, you will need to run:
-sudo systemctl stop nix-daemon.socket
-sudo systemctl stop nix-daemon.service
-sudo systemctl disable nix-daemon.socket
-sudo systemctl disable nix-daemon.service
-sudo systemctl daemon-reload
-
-# If you are on macOS, you will need to run:
-sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-</screen>
-
-    There may also be references to Nix in
-    <filename>/etc/profile</filename>,
-    <filename>/etc/bashrc</filename>, and
-    <filename>/etc/zshrc</filename> which you may remove.
-  </para>
-
-</section>
-
-<section xml:id="sect-macos-installation">
-  <title>macOS Installation</title>
-
-  <para>
-    Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
-    This means <filename>/nix</filename> can no longer live on your system
-    volume, and that you'll need a workaround to install Nix.
-  </para>
-
-  <para>
-    The recommended approach, which creates an unencrypted APFS volume
-    for your Nix store and a "synthetic" empty directory to mount it
-    over at <filename>/nix</filename>, is least likely to impair Nix
-    or your system.
-  </para>
-
-  <note><para>
-    With all separate-volume approaches, it's possible something on
-    your system (particularly daemons/services and restored apps) may
-    need access to your Nix store before the volume is mounted. Adding
-    additional encryption makes this more likely.
-  </para></note>
-
-  <para>
-    If you're using a recent Mac with a
-    <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
-    your drive will still be encrypted at rest (in which case "unencrypted"
-    is a bit of a misnomer). To use this approach, just install Nix with:
-  </para>
-
-  <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
-
-  <para>
-    If you don't like the sound of this, you'll want to weigh the
-    other approaches and tradeoffs detailed in this section.
-  </para>
-
-  <note>
-    <title>Eventual solutions?</title>
-    <para>
-      All of the known workarounds have drawbacks, but we hope
-      better solutions will be available in the future. Some that
-      we have our eye on are:
-    </para>
-    <orderedlist>
-      <listitem>
-        <para>
-          A true firmlink would enable the Nix store to live on the
-          primary data volume without the build problems caused by
-          the symlink approach. End users cannot currently
-          create true firmlinks.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          If the Nix store volume shared FileVault encryption
-          with the primary data volume (probably by using the same
-          volume group and role), FileVault encryption could be
-          easily supported by the installer without requiring
-          manual setup by each user.
-        </para>
-      </listitem>
-    </orderedlist>
-  </note>
-
-  <section xml:id="sect-macos-installation-change-store-prefix">
-    <title>Change the Nix store path prefix</title>
-    <para>
-      Changing the default prefix for the Nix store is a simple
-      approach which enables you to leave it on your root volume,
-      where it can take full advantage of FileVault encryption if
-      enabled. Unfortunately, this approach also opts your device out
-      of some benefits that are enabled by using the same prefix
-      across systems:
-
-      <itemizedlist>
-        <listitem>
-          <para>
-            Your system won't be able to take advantage of the binary
-            cache (unless someone is able to stand up and support
-            duplicate caching infrastructure), which means you'll
-            spend more time waiting for builds.
-          </para>
-        </listitem>
-        <listitem>
-          <para>
-            It's harder to build and deploy packages to Linux systems.
-          </para>
-        </listitem>
-        <!-- TODO: may be more here -->
-      </itemizedlist>
-
-      <!-- TODO: Yes, but how?! -->
-
-      It would also possible (and often requested) to just apply this
-      change ecosystem-wide, but it's an intrusive process that has
-      side effects we want to avoid for now.
-      <!-- magnificent hand-wavy gesture -->
-    </para>
-    <para>
-    </para>
-  </section>
-
-  <section xml:id="sect-macos-installation-encrypted-volume">
-    <title>Use a separate encrypted volume</title>
-    <para>
-      If you like, you can also add encryption to the recommended
-      approach taken by the installer. You can do this by pre-creating
-      an encrypted volume before you run the installer--or you can
-      run the installer and encrypt the volume it creates later.
-      <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
-    </para>
-    <para>
-      In either case, adding encryption to a second volume isn't quite
-      as simple as enabling FileVault for your boot volume. Before you
-      dive in, there are a few things to weigh:
-    </para>
-    <orderedlist>
-      <listitem>
-        <para>
-          The additional volume won't be encrypted with your existing
-          FileVault key, so you'll need another mechanism to decrypt
-          the volume.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          You can store the password in Keychain to automatically
-          decrypt the volume on boot--but it'll have to wait on Keychain
-          and may not mount before your GUI apps restore. If any of
-          your launchd agents or apps depend on Nix-installed software
-          (for example, if you use a Nix-installed login shell), the
-          restore may fail or break.
-        </para>
-        <para>
-          On a case-by-case basis, you may be able to work around this
-          problem by using <command>wait4path</command> to block
-          execution until your executable is available.
-        </para>
-        <para>
-          It's also possible to decrypt and mount the volume earlier
-          with a login hook--but this mechanism appears to be
-          deprecated and its future is unclear.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
-          You can hard-code the password in the clear, so that your
-          store volume can be decrypted before Keychain is available.
-        </para>
-      </listitem>
-    </orderedlist>
-    <para>
-      If you are comfortable navigating these tradeoffs, you can encrypt the volume with
-      something along the lines of:
-      <!-- TODO:
-      I don't know if this also needs from-scratch instructions?
-      can we just recommend use-the-installer-and-then-encrypt?
-      -->
-    </para>
-    <!--
-    TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
-
-    It may be more clear to use encryptVolume, here? FileVault seems
-    heavily associated with the boot-volume behavior; I worry
-    a little that it can mislead here, especially as it gets
-    copied around minus doc context...?
-    -->
-    <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
-
-    <!-- TODO: and then go into detail on the mount/decrypt approaches? -->
-  </section>
-
-  <section xml:id="sect-macos-installation-symlink">
-    <!--
-    Maybe a good razor is: if we'd hate having to support someone who
-    installed Nix this way, it shouldn't even be detailed?
-    -->
-    <title>Symlink the Nix store to a custom location</title>
-    <para>
-      Another simple approach is using <filename>/etc/synthetic.conf</filename>
-      to symlink the Nix store to the data volume. This option also
-      enables your store to share any configured FileVault encryption.
-      Unfortunately, builds that resolve the symlink may leak the
-      canonical path or even fail.
-    </para>
-    <para>
-      Because of these downsides, we can't recommend this approach.
-    </para>
-    <!-- Leaving out instructions for this one. -->
-  </section>
-
-  <section xml:id="sect-macos-installation-recommended-notes">
-    <title>Notes on the recommended approach</title>
-    <para>
-      This section goes into a little more detail on the recommended
-      approach. You don't need to understand it to run the installer,
-      but it can serve as a helpful reference if you run into trouble.
-    </para>
-    <orderedlist>
-      <listitem>
-        <para>
-          In order to compose user-writable locations into the new
-          read-only system root, Apple introduced a new concept called
-          <literal>firmlinks</literal>, which it describes as a
-          "bi-directional wormhole" between two filesystems. You can
-          see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
-          Unfortunately, firmlinks aren't (currently?) user-configurable.
-        </para>
-
-        <para>
-          For special cases like NFS mount points or package manager roots,
-          <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
-          supports limited user-controlled file-creation (of symlinks,
-          and synthetic empty directories) at <filename>/</filename>.
-          To create a synthetic empty directory for mounting at <filename>/nix</filename>,
-          add the following line to <filename>/etc/synthetic.conf</filename>
-          (create it if necessary):
-        </para>
-
-        <screen>nix</screen>
-      </listitem>
-
-      <listitem>
-        <para>
-          This configuration is applied at boot time, but you can use
-          <command>apfs.util</command> to trigger creation (not deletion)
-          of new entries without a reboot:
-        </para>
-
-        <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
-      </listitem>
-
-      <listitem>
-        <para>
-          Create the new APFS volume with diskutil:
-        </para>
-
-        <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
-      </listitem>
-
-      <listitem>
-        <para>
-          Using <command>vifs</command>, add the new mount to
-          <filename>/etc/fstab</filename>. If it doesn't already have
-          other entries, it should look something like:
-        </para>
-
-<screen>
-#
-# Warning - this file should only be modified with vifs(8)
-#
-# Failure to do so is unsupported and may be destructive.
-#
-LABEL=Nix\040Store /nix apfs rw,nobrowse
-</screen>
-
-        <para>
-          The nobrowse setting will keep Spotlight from indexing this
-          volume, and keep it from showing up on your desktop.
-        </para>
-      </listitem>
-    </orderedlist>
-  </section>
-
-</section>
-
-<section xml:id="sect-nix-install-pinned-version-url">
-  <title>Installing a pinned Nix version from a URL</title>
-
-  <para>
-    NixOS.org hosts version-specific installation URLs for all Nix
-    versions since 1.11.16, at
-    <literal>https://releases.nixos.org/nix/nix-<emphasis>version</emphasis>/install</literal>.
-  </para>
-
-  <para>
-    These install scripts can be used the same as the main
-  NixOS.org installation script:
-
-  <screen>
-  sh &lt;(curl -L https://nixos.org/nix/install)
-</screen>
-  </para>
-
-  <para>
-    In the same directory of the install script are sha256 sums, and
-    gpg signature files.
-  </para>
-</section>
-
-<section xml:id="sect-nix-install-binary-tarball">
-  <title>Installing from a binary tarball</title>
-
-  <para>
-    You can also download a binary tarball that contains Nix and all
-    its dependencies.  (This is what the install script at
-    <uri>https://nixos.org/nix/install</uri> does automatically.)  You
-    should unpack it somewhere (e.g. in <filename>/tmp</filename>),
-    and then run the script named <command>install</command> inside
-    the binary tarball:
-
-
-<screen>
-alice$ cd /tmp
-alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-alice$ cd nix-1.8-x86_64-darwin
-alice$ ./install
-</screen>
-  </para>
-
-  <para>
-    If you need to edit the multi-user installation script to use
-    different group ID or a different user ID range, modify the
-    variables set in the file named
-    <filename>install-multi-user</filename>.
-  </para>
-</section>
-</chapter>
diff --git a/doc/manual/installation/installing-source.xml b/doc/manual/installation/installing-source.xml
deleted file mode 100644
index c261a109d..000000000
--- a/doc/manual/installation/installing-source.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-installing-source">
-
-<title>Installing Nix from Source</title>
-
-<para>If no binary package is available, you can download and compile
-a source distribution.</para>
-
-<xi:include href="prerequisites-source.xml" />
-<xi:include href="obtaining-source.xml" />
-<xi:include href="building-source.xml" />
-
-</chapter>
diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml
deleted file mode 100644
index 835bd3a52..000000000
--- a/doc/manual/installation/multi-user.xml
+++ /dev/null
@@ -1,107 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-multi-user">
-
-<title>Multi-User Mode</title>
-
-<para>To allow a Nix store to be shared safely among multiple users,
-it is important that users are not able to run builders that modify
-the Nix store or database in arbitrary ways, or that interfere with
-builds started by other users.  If they could do so, they could
-install a Trojan horse in some package and compromise the accounts of
-other users.</para>
-
-<para>To prevent this, the Nix store and database are owned by some
-privileged user (usually <literal>root</literal>) and builders are
-executed under special user accounts (usually named
-<literal>nixbld1</literal>, <literal>nixbld2</literal>, etc.).  When a
-unprivileged user runs a Nix command, actions that operate on the Nix
-store (such as builds) are forwarded to a <emphasis>Nix
-daemon</emphasis> running under the owner of the Nix store/database
-that performs the operation.</para>
-
-<note><para>Multi-user mode has one important limitation: only
-<systemitem class="username">root</systemitem> and a set of trusted
-users specified in <filename>nix.conf</filename> can specify arbitrary
-binary caches. So while unprivileged users may install packages from
-arbitrary Nix expressions, they may not get pre-built
-binaries.</para></note>
-
-
-<section>
-
-<title>Setting up the build users</title>
-
-<para>The <emphasis>build users</emphasis> are the special UIDs under
-which builds are performed.  They should all be members of the
-<emphasis>build users group</emphasis> <literal>nixbld</literal>.
-This group should have no other members.  The build users should not
-be members of any other group. On Linux, you can create the group and
-users as follows:
-
-<screen>
-$ groupadd -r nixbld
-$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \
-    -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \
-    nixbld$n; done
-</screen>
-
-This creates 10 build users. There can never be more concurrent builds
-than the number of build users, so you may want to increase this if
-you expect to do many builds at the same time.</para>
-
-</section>
-
-
-<section>
-
-<title>Running the daemon</title>
-
-<para>The <link linkend="sec-nix-daemon">Nix daemon</link> should be
-started as follows (as <literal>root</literal>):
-
-<screen>
-$ nix-daemon</screen>
-
-You’ll want to put that line somewhere in your system’s boot
-scripts.</para>
-
-<para>To let unprivileged users use the daemon, they should set the
-<link linkend="envar-remote"><literal>NIX_REMOTE</literal> environment
-variable</link> to <literal>daemon</literal>.  So you should put a
-line like
-
-<programlisting>
-export NIX_REMOTE=daemon</programlisting>
-
-into the users’ login scripts.</para>
-
-</section>
-
-
-<section>
-
-<title>Restricting access</title>
-
-<para>To limit which users can perform Nix operations, you can use the
-permissions on the directory
-<filename>/nix/var/nix/daemon-socket</filename>.  For instance, if you
-want to restrict the use of Nix to the members of a group called
-<literal>nix-users</literal>, do
-
-<screen>
-$ chgrp nix-users /nix/var/nix/daemon-socket
-$ chmod ug=rwx,o= /nix/var/nix/daemon-socket
-</screen>
-
-This way, users who are not in the <literal>nix-users</literal> group
-cannot connect to the Unix domain socket
-<filename>/nix/var/nix/daemon-socket/socket</filename>, so they cannot
-perform Nix operations.</para>
-
-</section>
-
-
-</section>
diff --git a/doc/manual/installation/nix-security.xml b/doc/manual/installation/nix-security.xml
deleted file mode 100644
index d888ff14d..000000000
--- a/doc/manual/installation/nix-security.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-nix-security">
-
-<title>Security</title>
-
-<para>Nix has two basic security models.  First, it can be used in
-“single-user mode”, which is similar to what most other package
-management tools do: there is a single user (typically <systemitem
-class="username">root</systemitem>) who performs all package
-management operations.  All other users can then use the installed
-packages, but they cannot perform package management operations
-themselves.</para>
-
-<para>Alternatively, you can configure Nix in “multi-user mode”.  In
-this model, all users can perform package management operations — for
-instance, every user can install software without requiring root
-privileges.  Nix ensures that this is secure.  For instance, it’s not
-possible for one user to overwrite a package used by another user with
-a Trojan horse.</para>
-
-<xi:include href="single-user.xml" />
-<xi:include href="multi-user.xml" />
-
-</chapter>
\ No newline at end of file
diff --git a/doc/manual/installation/obtaining-source.xml b/doc/manual/installation/obtaining-source.xml
deleted file mode 100644
index 968822cc0..000000000
--- a/doc/manual/installation/obtaining-source.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-obtaining-source">
-
-<title>Obtaining a Source Distribution</title>
-
-<para>The source tarball of the most recent stable release can be
-downloaded from the <link
-xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>.
-You can also grab the <link
-xlink:href="http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents">most
-recent development release</link>.</para>
-
-<para>Alternatively, the most recent sources of Nix can be obtained
-from its <link
-xlink:href="https://github.com/NixOS/nix">Git
-repository</link>.  For example, the following command will check out
-the latest revision into a directory called
-<filename>nix</filename>:</para>
-
-<screen>
-$ git clone https://github.com/NixOS/nix</screen>
-
-<para>Likewise, specific releases can be obtained from the <link
-xlink:href="https://github.com/NixOS/nix/tags">tags</link> of the
-repository.</para>
-
-</section>
\ No newline at end of file
diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
deleted file mode 100644
index 77955eecc..000000000
--- a/doc/manual/installation/prerequisites-source.xml
+++ /dev/null
@@ -1,101 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-prerequisites-source">
-
-<title>Prerequisites</title>
-
-<itemizedlist>
-
-  <listitem><para>GNU Autoconf
-  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
-  and the autoconf-archive macro collection
-  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
-  These are only needed to run the bootstrap script, and are not necessary
-  if your source distribution came with a pre-built
-  <literal>./configure</literal> script.</para></listitem>
-
-  <listitem><para>GNU Make.</para></listitem>
-  
-  <listitem><para>Bash Shell. The <literal>./configure</literal> script
-  relies on bashisms, so Bash is required.</para></listitem>
-
-  <listitem><para>A version of GCC or Clang that supports C++17.</para></listitem>
-
-  <listitem><para><command>pkg-config</command> to locate
-  dependencies.  If your distribution does not provide it, you can get
-  it from <link
-  xlink:href="http://www.freedesktop.org/wiki/Software/pkg-config"
-  />.</para></listitem>
-
-  <listitem><para>The OpenSSL library to calculate cryptographic hashes.
-  If your distribution does not provide it, you can get it from <link
-  xlink:href="https://www.openssl.org"/>.</para></listitem>
-
-  <listitem><para>The <literal>libbrotlienc</literal> and
-  <literal>libbrotlidec</literal> libraries to provide implementation
-  of the Brotli compression algorithm. They are available for download
-  from the official repository <link
-  xlink:href="https://github.com/google/brotli" />.</para></listitem>
-
-  <listitem><para>The bzip2 compressor program and the
-  <literal>libbz2</literal> library.  Thus you must have bzip2
-  installed, including development headers and libraries.  If your
-  distribution does not provide these, you can obtain bzip2 from <link
-  xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/"
-  />.</para></listitem>
-
-  <listitem><para><literal>liblzma</literal>, which is provided by
-  XZ Utils. If your distribution does not provide this, you can
-  get it from <link xlink:href="https://tukaani.org/xz/"/>.</para></listitem>
-  
-  <listitem><para>cURL and its library. If your distribution does not
-  provide it, you can get it from <link
-  xlink:href="https://curl.haxx.se/"/>.</para></listitem>
-      
-  <listitem><para>The SQLite embedded database library, version 3.6.19
-  or higher.  If your distribution does not provide it, please install
-  it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
-
-  <listitem><para>The <link
-  xlink:href="http://www.hboehm.info/gc/">Boehm
-  garbage collector</link> to reduce the evaluator’s memory
-  consumption (optional).  To enable it, install
-  <literal>pkgconfig</literal> and the Boehm garbage collector, and
-  pass the flag <option>--enable-gc</option> to
-  <command>configure</command>.</para></listitem>
-
-  <listitem><para>The <literal>boost</literal> library of version
-  1.66.0 or higher. It can be obtained from the official web site
-  <link xlink:href="https://www.boost.org/" />.</para></listitem>
-
-  <listitem><para>The <literal>editline</literal> library of version
-  1.14.0 or higher. It can be obtained from the its repository
-  <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem>
-
-  <listitem><para>Recent versions of Bison and Flex to build the
-  parser.  (This is because Nix needs GLR support in Bison and
-  reentrancy support in Flex.)  For Bison, you need version 2.6, which
-  can be obtained from the <link
-  xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP
-  server</link>.  For Flex, you need version 2.5.35, which is
-  available on <link
-  xlink:href="http://lex.sourceforge.net/">SourceForge</link>.
-  Slightly older versions may also work, but ancient versions like the
-  ubiquitous 2.5.4a won't.  Note that these are only required if you
-  modify the parser or when you are building from the Git
-  repository.</para></listitem>
-
-  <listitem><para>The <literal>libseccomp</literal> is used to provide
-  syscall filtering on Linux. This is an optional dependency and can
-  be disabled passing a <option>--disable-seccomp-sandboxing</option>
-  option to the <command>configure</command> script (Not recommended
-  unless your system doesn't support
-  <literal>libseccomp</literal>). To get the library, visit <link
-  xlink:href="https://github.com/seccomp/libseccomp"
-  />.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/installation/single-user.xml b/doc/manual/installation/single-user.xml
deleted file mode 100644
index e9a761af3..000000000
--- a/doc/manual/installation/single-user.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-single-user">
-
-<title>Single-User Mode</title>
-
-<para>In single-user mode, all Nix operations that access the database
-in <filename><emphasis>prefix</emphasis>/var/nix/db</filename>
-or modify the Nix store in
-<filename><emphasis>prefix</emphasis>/store</filename> must be
-performed under the user ID that owns those directories.  This is
-typically <systemitem class="username">root</systemitem>.  (If you
-install from RPM packages, that’s in fact the default ownership.)
-However, on single-user machines, it is often convenient to
-<command>chown</command> those directories to your normal user account
-so that you don’t have to <command>su</command> to <systemitem
-class="username">root</systemitem> all the time.</para>
-
-</section>
\ No newline at end of file
diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml
deleted file mode 100644
index 3e74be49d..000000000
--- a/doc/manual/installation/supported-platforms.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-supported-platforms">
-
-<title>Supported Platforms</title>
-
-<para>Nix is currently supported on the following platforms:
-
-<itemizedlist>
-
-  <listitem><para>Linux (i686, x86_64, aarch64).</para></listitem>
-
-  <listitem><para>macOS (x86_64).</para></listitem>
-
-  <!--
-  <listitem><para>FreeBSD (only tested on Intel).</para></listitem>
-  -->
-
-  <!--
-  <listitem><para>Windows through <link
-  xlink:href="http://www.cygwin.com/">Cygwin</link>.</para>
-
-  <warning><para>On Cygwin, Nix <emphasis>must</emphasis> be installed
-  on an NTFS partition.  It will not work correctly on a FAT
-  partition.</para></warning>
-
-  </listitem>
-  -->
-
-</itemizedlist>
-
-</para>
-
-</chapter>
diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml
deleted file mode 100644
index 592f63895..000000000
--- a/doc/manual/installation/upgrading.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-upgrading-nix">
-
-  <title>Upgrading Nix</title>
-
-  <para>
-    Multi-user Nix users on macOS can upgrade Nix by running:
-    <command>sudo -i sh -c 'nix-channel --update &amp;&amp;
-    nix-env -iA nixpkgs.nix &amp;&amp;
-    launchctl remove org.nixos.nix-daemon &amp;&amp;
-    launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command>
-  </para>
-
-
-  <para>
-    Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
-  </para>
-      
-  <para>
-    Multi-user Nix users on Linux should run this with sudo:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
-  </para>
-</chapter>
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
deleted file mode 100644
index 039185703..000000000
--- a/doc/manual/introduction/about-nix.xml
+++ /dev/null
@@ -1,268 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-about-nix">
-
-<title>About Nix</title>
-
-<para>Nix is a <emphasis>purely functional package manager</emphasis>.
-This means that it treats packages like values in purely functional
-programming languages such as Haskell — they are built by functions
-that don’t have side-effects, and they never change after they have
-been built.  Nix stores packages in the <emphasis>Nix
-store</emphasis>, usually the directory
-<filename>/nix/store</filename>, where each package has its own unique
-subdirectory such as
-
-<programlisting>
-/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/
-</programlisting>
-
-where <literal>b6gvzjyb2pg0…</literal> is a unique identifier for the
-package that captures all its dependencies (it’s a cryptographic hash
-of the package’s build dependency graph).  This enables many powerful
-features.</para>
-
-
-<section><title>Multiple versions</title>
-
-<para>You can have multiple versions or variants of a package
-installed at the same time.  This is especially important when
-different applications have dependencies on different versions of the
-same package — it prevents the “DLL hell”.  Because of the hashing
-scheme, different versions of a package end up in different paths in
-the Nix store, so they don’t interfere with each other.</para>
-
-<para>An important consequence is that operations like upgrading or
-uninstalling an application cannot break other applications, since
-these operations never “destructively” update or delete files that are
-used by other packages.</para>
-
-</section>
-
-
-<section><title>Complete dependencies</title>
-
-<para>Nix helps you make sure that package dependency specifications
-are complete.  In general, when you’re making a package for a package
-management system like RPM, you have to specify for each package what
-its dependencies are, but there are no guarantees that this
-specification is complete.  If you forget a dependency, then the
-package will build and work correctly on <emphasis>your</emphasis>
-machine if you have the dependency installed, but not on the end
-user's machine if it's not there.</para>
-
-<para>Since Nix on the other hand doesn’t install packages in “global”
-locations like <filename>/usr/bin</filename> but in package-specific
-directories, the risk of incomplete dependencies is greatly reduced.
-This is because tools such as compilers don’t search in per-packages
-directories such as
-<filename>/nix/store/5lbfaxb722zp…-openssl-0.9.8d/include</filename>,
-so if a package builds correctly on your system, this is because you
-specified the dependency explicitly. This takes care of the build-time
-dependencies.</para>
-
-<para>Once a package is built, runtime dependencies are found by
-scanning binaries for the hash parts of Nix store paths (such as
-<literal>r8vvq9kq…</literal>).  This sounds risky, but it works
-extremely well.</para>
-
-</section>
-
-
-<section><title>Multi-user support</title>
-
-<para>Nix has multi-user support.  This means that non-privileged
-users can securely install software.  Each user can have a different
-<emphasis>profile</emphasis>, a set of packages in the Nix store that
-appear in the user’s <literal>PATH</literal>.  If a user installs a
-package that another user has already installed previously, the
-package won’t be built or downloaded a second time.  At the same time,
-it is not possible for one user to inject a Trojan horse into a
-package that might be used by another user.</para>
-
-</section>
-
-
-<section><title>Atomic upgrades and rollbacks</title>
-
-<para>Since package management operations never overwrite packages in
-the Nix store but just add new versions in different paths, they are
-<emphasis>atomic</emphasis>.  So during a package upgrade, there is no
-time window in which the package has some files from the old version
-and some files from the new version — which would be bad because a
-program might well crash if it’s started during that period.</para>
-
-<para>And since packages aren’t overwritten, the old versions are still
-there after an upgrade.  This means that you can <emphasis>roll
-back</emphasis> to the old version:</para>
-
-<screen>
-$ nix-env --upgrade <emphasis>some-packages</emphasis>
-$ nix-env --rollback
-</screen>
-
-</section>
-
-
-<section><title>Garbage collection</title>
-
-<para>When you uninstall a package like this…
-
-<screen>
-$ nix-env --uninstall firefox
-</screen>
-
-the package isn’t deleted from the system right away (after all, you
-might want to do a rollback, or it might be in the profiles of other
-users).  Instead, unused packages can be deleted safely by running the
-<emphasis>garbage collector</emphasis>:
-
-<screen>
-$ nix-collect-garbage
-</screen>
-
-This deletes all packages that aren’t in use by any user profile or by
-a currently running program.</para>
-
-</section>
-
-
-<section><title>Functional package language</title>
-
-<para>Packages are built from <emphasis>Nix expressions</emphasis>,
-which is a simple functional language.  A Nix expression describes
-everything that goes into a package build action (a “derivation”):
-other packages, sources, the build script, environment variables for
-the build script, etc.  Nix tries very hard to ensure that Nix
-expressions are <emphasis>deterministic</emphasis>: building a Nix
-expression twice should yield the same result.</para>
-
-<para>Because it’s a functional language, it’s easy to support
-building variants of a package: turn the Nix expression into a
-function and call it any number of times with the appropriate
-arguments.  Due to the hashing scheme, variants don’t conflict with
-each other in the Nix store.</para>
-
-</section>
-
-
-<section><title>Transparent source/binary deployment</title>
-
-<para>Nix expressions generally describe how to build a package from
-source, so an installation action like
-
-<screen>
-$ nix-env --install firefox
-</screen>
-
-<emphasis>could</emphasis> cause quite a bit of build activity, as not
-only Firefox but also all its dependencies (all the way up to the C
-library and the compiler) would have to built, at least if they are
-not already in the Nix store.  This is a <emphasis>source deployment
-model</emphasis>.  For most users, building from source is not very
-pleasant as it takes far too long.  However, Nix can automatically
-skip building from source and instead use a <emphasis>binary
-cache</emphasis>, a web server that provides pre-built binaries. For
-instance, when asked to build
-<literal>/nix/store/b6gvzjyb2pg0…-firefox-33.1</literal> from source,
-Nix would first check if the file
-<uri>https://cache.nixos.org/b6gvzjyb2pg0….narinfo</uri> exists, and
-if so, fetch the pre-built binary referenced from there; otherwise, it
-would fall back to building from source.</para>
-
-</section>
-
-
-<!--
-<section><title>Binary patching</title>
-
-<para>In addition to downloading binaries automatically if they’re
-available, Nix can download binary deltas that patch an existing
-package in the Nix store into a new version.  This speeds up
-upgrades.</para>
-
-</section>
--->
-
-
-<section><title>Nix Packages collection</title>
-
-<para>We provide a large set of Nix expressions containing hundreds of
-existing Unix packages, the <emphasis>Nix Packages
-collection</emphasis> (Nixpkgs).</para>
-
-</section>
-
-
-<section><title>Managing build environments</title>
-
-<para>Nix is extremely useful for developers as it makes it easy to
-automatically set up the build environment for a package. Given a
-Nix expression that describes the dependencies of your package, the
-command <command>nix-shell</command> will build or download those
-dependencies if they’re not already in your Nix store, and then start
-a Bash shell in which all necessary environment variables (such as
-compiler search paths) are set.</para>
-
-<para>For example, the following command gets all dependencies of the
-Pan newsreader, as described by <link
-xlink:href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix">its
-Nix expression</link>:</para>
-
-<screen>
-$ nix-shell '&lt;nixpkgs>' -A pan
-</screen>
-
-<para>You’re then dropped into a shell where you can edit, build and test
-the package:</para>
-
-<screen>
-[nix-shell]$ tar xf $src
-[nix-shell]$ cd pan-*
-[nix-shell]$ ./configure
-[nix-shell]$ make
-[nix-shell]$ ./pan/gui/pan
-</screen>
-
-<!--
-<para>Since Nix packages are reproducible and have complete dependency
-specifications, Nix makes an excellent basis for <a
-href="[%root%]hydra">a continuous build system</a>.</para>
--->
-
-</section>
-
-
-<section><title>Portability</title>
-
-<para>Nix runs on Linux and macOS.</para>
-
-</section>
-
-
-<section><title>NixOS</title>
-
-<para>NixOS is a Linux distribution based on Nix.  It uses Nix not
-just for package management but also to manage the system
-configuration (e.g., to build configuration files in
-<filename>/etc</filename>).  This means, among other things, that it
-is easy to roll back the entire configuration of the system to an
-earlier state.  Also, users can install software without root
-privileges.  For more information and downloads, see the <link
-xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
-
-</section>
-
-
-<section><title>License</title>
-
-<para>Nix is released under the terms of the <link
-xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
-LGPLv2.1 or (at your option) any later version</link>.</para>
-
-</section>
-
-
-</chapter>
diff --git a/doc/manual/introduction/introduction.xml b/doc/manual/introduction/introduction.xml
deleted file mode 100644
index 12b2cc761..000000000
--- a/doc/manual/introduction/introduction.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<part xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="chap-introduction">
-
-<title>Introduction</title>
-
-<xi:include href="about-nix.xml" />
-<xi:include href="quick-start.xml" />
-
-</part>
diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml
deleted file mode 100644
index 5d9a55e4e..000000000
--- a/doc/manual/introduction/quick-start.xml
+++ /dev/null
@@ -1,124 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="chap-quick-start">
-
-<title>Quick Start</title>
-
-<para>This chapter is for impatient people who don't like reading
-documentation.  For more in-depth information you are kindly referred
-to subsequent chapters.</para>
-
-<procedure>
-
-<step><para>Install single-user Nix by running the following:
-
-<screen>
-$ bash &lt;(curl -L https://nixos.org/nix/install)
-</screen>
-
-This will install Nix in <filename>/nix</filename>. The install script
-will create <filename>/nix</filename> using <command>sudo</command>,
-so make sure you have sufficient rights.  (For other installation
-methods, see <xref linkend="chap-installation"/>.)</para></step>
-
-<step><para>See what installable packages are currently available
-in the channel:
-
-<screen>
-$ nix-env -qa
-docbook-xml-4.3
-docbook-xml-4.5
-firefox-33.0.2
-hello-2.9
-libxslt-1.1.28
-<emphasis>...</emphasis></screen>
-
-</para></step>
-
-<step><para>Install some packages from the channel:
-
-<screen>
-$ nix-env -i hello</screen>
-
-This should download pre-built packages; it should not build them
-locally (if it does, something went wrong).</para></step>
-
-<step><para>Test that they work:
-
-<screen>
-$ which hello
-/home/eelco/.nix-profile/bin/hello
-$ hello
-Hello, world!
-</screen>
-
-</para></step>
-
-<step><para>Uninstall a package:
-
-<screen>
-$ nix-env -e hello</screen>
-
-</para></step>
-
-<step><para>You can also test a package without installing it:
-
-<screen>
-$ nix-shell -p hello
-</screen>
-
-This builds or downloads GNU Hello and its dependencies, then drops
-you into a Bash shell where the <command>hello</command> command is
-present, all without affecting your normal environment:
-
-<screen>
-[nix-shell:~]$ hello
-Hello, world!
-
-[nix-shell:~]$ exit
-
-$ hello
-hello: command not found
-</screen>
-
-</para></step>
-
-<step><para>To keep up-to-date with the channel, do:
-
-<screen>
-$ nix-channel --update nixpkgs
-$ nix-env -u '*'</screen>
-
-The latter command will upgrade each installed package for which there
-is a “newer” version (as determined by comparing the version
-numbers).</para></step>
-
-<step><para>If you're unhappy with the result of a
-<command>nix-env</command> action (e.g., an upgraded package turned
-out not to work properly), you can go back:
-
-<screen>
-$ nix-env --rollback</screen>
-
-</para></step>
-
-<step><para>You should periodically run the Nix garbage collector
-to get rid of unused packages, since uninstalls or upgrades don't
-actually delete them:
-
-<screen>
-$ nix-collect-garbage -d</screen>
-
-<!--
-The first command deletes old “generations” of your profile (making
-rollbacks impossible, but also making the packages in those old
-generations available for garbage collection), while the second
-command actually deletes them.-->
-
-</para></step>
-
-</procedure>
-
-</chapter>
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 13d937f8d..2d730a60e 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -2,14 +2,6 @@ ifeq ($(doc_generate),yes)
 
 MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
 
-#$(d)/version.txt:
-#	$(trace-gen) echo -n $(PACKAGE_VERSION) > $@
-
-clean-files += $(d)/version.txt
-
-dist-files += $(d)/version.txt
-
-
 # Generate man pages.
 man-pages := $(foreach n, \
   nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
@@ -38,5 +30,4 @@ install: $(docdir)/manual/index.html
 $(docdir)/manual/index.html: $(MANUAL_SRCS)
 	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
 
-
 endif
diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml
deleted file mode 100644
index 87d9de28a..000000000
--- a/doc/manual/manual.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-<book xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0">
-
-  <info>
-    <title>Nix Package Manager Guide</title>
-    <subtitle>Version <xi:include href="version.txt" parse="text" /></subtitle>
-
-    <author>
-      <personname>
-        <firstname>Eelco</firstname>
-        <surname>Dolstra</surname>
-      </personname>
-      <contrib>Author</contrib>
-    </author>
-
-    <copyright>
-      <year>2004-2018</year>
-      <holder>Eelco Dolstra</holder>
-    </copyright>
-
-  </info>
-
-  <!--
-  <preface>
-    <title>Preface</title>
-    <para>This manual describes how to set up and use the Nix package
-    manager.</para>
-  </preface>
-  -->
-
-  <xi:include href="introduction/introduction.xml" />
-  <xi:include href="installation/installation.xml" />
-  <xi:include href="installation/upgrading.xml" />
-  <xi:include href="packages/package-management.xml" />
-  <xi:include href="expressions/writing-nix-expressions.xml" />
-  <xi:include href="advanced-topics/advanced-topics.xml" />
-  <xi:include href="command-ref/command-ref.xml" />
-  <xi:include href="glossary/glossary.xml" />
-  <xi:include href="hacking.xml" />
-  <xi:include href="release-notes/release-notes.xml" />
-
-<!--
-<appendix>
-    <title>Nix Release Notes</title>
-    <xi:include href="release-notes/release-notes.xml"
-                xpointer="xmlns(x=http://docbook.org/ns/docbook)xpointer(x:article/x:section)" />
-  </appendix>
--->
-
-</book>
diff --git a/doc/manual/nix-lang-ref.xml b/doc/manual/nix-lang-ref.xml
deleted file mode 100644
index 86273ac3d..000000000
--- a/doc/manual/nix-lang-ref.xml
+++ /dev/null
@@ -1,182 +0,0 @@
-<appendix>
-  <title>Nix Language Reference</title>
-
-  <sect1>
-    <title>Grammar</title>
-
-    <productionset>
-      <title>Expressions</title>
-      
-      <production id="nix.expr">
-        <lhs>Expr</lhs>
-        <rhs>
-          <nonterminal def="#nix.expr_function" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_function">
-        <lhs>ExprFunction</lhs>
-        <rhs>
-          '{' <nonterminal def="#nix.formals" /> '}' ':' <nonterminal def="#nix.expr_function" />
-          <sbr />|
-          <nonterminal def="#nix.expr_assert" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_assert">
-        <lhs>ExprAssert</lhs>
-        <rhs>
-          'assert' <nonterminal def="#nix.expr" /> ';' <nonterminal def="#nix.expr_assert" />
-          <sbr />|
-          <nonterminal def="#nix.expr_if" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_if">
-        <lhs>ExprIf</lhs>
-        <rhs>
-          'if' <nonterminal def="#nix.expr" /> 'then' <nonterminal def="#nix.expr" />
-          'else' <nonterminal def="#nix.expr" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_op">
-        <lhs>ExprOp</lhs>
-        <rhs>
-          '!' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '==' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '!=' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '&amp;&amp;' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '||' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '->' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '//' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '~' <nonterminal def="#nix.expr_op" />
-          <sbr />|
-          <nonterminal def="#nix.expr_op" /> '?' <nonterminal def="#nix.id" />
-          <sbr />|
-          <nonterminal def="#nix.expr_app" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_app">
-        <lhs>ExprApp</lhs>
-        <rhs>
-          <nonterminal def="#nix.expr_app" /> '.' <nonterminal def="#nix.expr_select" />
-          <sbr />|
-          <nonterminal def="#nix.expr_select" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_select">
-        <lhs>ExprSelect</lhs>
-        <rhs>
-          <nonterminal def="#nix.expr_select" /> <nonterminal def="#nix.id" />
-          <sbr />|
-          <nonterminal def="#nix.expr_simple" />
-        </rhs>
-      </production>
-      
-      <production id="nix.expr_simple">
-        <lhs>ExprSimple</lhs>
-        <rhs>
-          <nonterminal def="#nix.id" /> |
-          <nonterminal def="#nix.int" /> |
-          <nonterminal def="#nix.str" /> |
-          <nonterminal def="#nix.path" /> |
-          <nonterminal def="#nix.uri" />
-          <sbr />|
-          'true' | 'false' | 'null'
-          <sbr />|
-          '(' <nonterminal def="#nix.expr" /> ')'
-          <sbr />|
-          '{' <nonterminal def="#nix.bind" />* '}'
-          <sbr />|
-          'let' '{' <nonterminal def="#nix.bind" />* '}'
-          <sbr />|
-          'rec' '{' <nonterminal def="#nix.bind" />* '}'
-          <sbr />|
-          '[' <nonterminal def="#nix.expr_select" />* ']'
-        </rhs>
-      </production>
-
-      <production id="nix.bind">
-        <lhs>Bind</lhs>
-        <rhs>
-          <nonterminal def="#nix.id" /> '=' <nonterminal def="#nix.expr" /> ';'
-          <sbr />|
-          'inherit' ('(' <nonterminal def="#nix.expr" /> ')')? <nonterminal def="#nix.id" />* ';'
-        </rhs>
-      </production>
-
-      <production id="nix.formals">
-        <lhs>Formals</lhs>
-        <rhs>
-          <nonterminal def="#nix.formal" /> ',' <nonterminal def="#nix.formals" />
-          | <nonterminal def="#nix.formal" />
-        </rhs>
-      </production>
-          
-      <production id="nix.formal">
-        <lhs>Formal</lhs>
-        <rhs>
-          <nonterminal def="#nix.id" />
-          <sbr />|
-          <nonterminal def="#nix.id" /> '?' <nonterminal def="#nix.expr" />
-        </rhs>
-      </production>
-          
-    </productionset>
-
-    <productionset>
-      <title>Terminals</title>
-
-      <production id="nix.id">
-        <lhs>Id</lhs>
-        <rhs>[a-zA-Z\_][a-zA-Z0-9\_\']*</rhs>
-      </production>
-    
-      <production id="nix.int">
-        <lhs>Int</lhs>
-        <rhs>[0-9]+</rhs>
-      </production>
-    
-      <production id="nix.str">
-        <lhs>Str</lhs>
-        <rhs>\"[^\n\"]*\"</rhs>
-      </production>
-
-      <production id="nix.path">
-        <lhs>Path</lhs>
-        <rhs>[a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+</rhs>
-      </production>
-    
-      <production id="nix.uri">
-        <lhs>Uri</lhs>
-        <rhs>[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&amp;\=\+\$\,\-\_\.\!\~\*\']+</rhs>
-      </production>
-
-      <production id="nix.ws">
-        <lhs>Whitespace</lhs>
-        <rhs>
-          [ \t\n]+
-          <sbr />|
-          \#[^\n]*
-          <sbr />|
-          \/\*(.|\n)*\*\/
-        </rhs>
-      </production>
-
-    </productionset>
-    
-  </sect1>
-
-</appendix>
diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml
deleted file mode 100644
index b8a2c7ab3..000000000
--- a/doc/manual/packages/basic-package-mgmt.xml
+++ /dev/null
@@ -1,194 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-basic-package-mgmt">
-
-<title>Basic Package Management</title>
-
-<para>The main command for package management is <link
-linkend="sec-nix-env"><command>nix-env</command></link>.  You can use
-it to install, upgrade, and erase packages, and to query what
-packages are installed or are available for installation.</para>
-
-<para>In Nix, different users can have different “views”
-on the set of installed applications.  That is, there might be lots of
-applications present on the system (possibly in many different
-versions), but users can have a specific selection of those active —
-where “active” just means that it appears in a directory
-in the user’s <literal>PATH</literal>.  Such a view on the set of
-installed applications is called a <emphasis>user
-environment</emphasis>, which is just a directory tree consisting of
-symlinks to the files of the active applications.  </para>
-
-<para>Components are installed from a set of <emphasis>Nix
-expressions</emphasis> that tell Nix how to build those packages,
-including, if necessary, their dependencies.  There is a collection of
-Nix expressions called the Nixpkgs package collection that contains
-packages ranging from basic development stuff such as GCC and Glibc,
-to end-user applications like Mozilla Firefox.  (Nix is however not
-tied to the Nixpkgs package collection; you could write your own Nix
-expressions based on Nixpkgs, or completely new ones.)</para>
-
-<para>You can manually download the latest version of Nixpkgs from
-<link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However,
-it’s much more convenient to use the Nixpkgs
-<emphasis>channel</emphasis>, since it makes it easy to stay up to
-date with new versions of Nixpkgs. (Channels are described in more
-detail in <xref linkend="sec-channels"/>.) Nixpkgs is automatically
-added to your list of “subscribed” channels when you install
-Nix. If this is not the case for some reason, you can add it as
-follows:
-
-<screen>
-$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-$ nix-channel --update
-</screen>
-
-</para>
-
-<note><para>On NixOS, you’re automatically subscribed to a NixOS
-channel corresponding to your NixOS major release
-(e.g. <uri>http://nixos.org/channels/nixos-14.12</uri>). A NixOS
-channel is identical to the Nixpkgs channel, except that it contains
-only Linux binaries and is updated only if a set of regression tests
-succeed.</para></note>
-
-<para>You can view the set of available packages in Nixpkgs:
-
-<screen>
-$ nix-env -qa
-aterm-2.2
-bash-3.0
-binutils-2.15
-bison-1.875d
-blackdown-1.4.2
-bzip2-1.0.2
-…</screen>
-
-The flag <option>-q</option> specifies a query operation, and
-<option>-a</option> means that you want to show the “available” (i.e.,
-installable) packages, as opposed to the installed packages. If you
-downloaded Nixpkgs yourself, or if you checked it out from GitHub,
-then you need to pass the path to your Nixpkgs tree using the
-<option>-f</option> flag:
-
-<screen>
-$ nix-env -qaf <emphasis>/path/to/nixpkgs</emphasis>
-</screen>
-
-where <emphasis>/path/to/nixpkgs</emphasis> is where you’ve
-unpacked or checked out Nixpkgs.</para>
-
-<para>You can select specific packages by name:
-
-<screen>
-$ nix-env -qa firefox
-firefox-34.0.5
-firefox-with-plugins-34.0.5
-</screen>
-
-and using regular expressions:
-
-<screen>
-$ nix-env -qa 'firefox.*'
-</screen>
-
-</para>
-
-<para>It is also possible to see the <emphasis>status</emphasis> of
-available packages, i.e., whether they are installed into the user
-environment and/or present in the system:
-
-<screen>
-$ nix-env -qas
-…
--PS bash-3.0
---S binutils-2.15
-IPS bison-1.875d
-…</screen>
-
-The first character (<literal>I</literal>) indicates whether the
-package is installed in your current user environment.  The second
-(<literal>P</literal>) indicates whether it is present on your system
-(in which case installing it into your user environment would be a
-very quick operation).  The last one (<literal>S</literal>) indicates
-whether there is a so-called <emphasis>substitute</emphasis> for the
-package, which is Nix’s mechanism for doing binary deployment.  It
-just means that Nix knows that it can fetch a pre-built package from
-somewhere (typically a network server) instead of building it
-locally.</para>
-
-<para>You can install a package using <literal>nix-env -i</literal>.
-For instance,
-
-<screen>
-$ nix-env -i subversion</screen>
-
-will install the package called <literal>subversion</literal> (which
-is, of course, the <link
-xlink:href='http://subversion.tigris.org/'>Subversion version
-management system</link>).</para>
-
-<note><para>When you ask Nix to install a package, it will first try
-to get it in pre-compiled form from a <emphasis>binary
-cache</emphasis>. By default, Nix will use the binary cache
-<uri>https://cache.nixos.org</uri>; it contains binaries for most
-packages in Nixpkgs. Only if no binary is available in the binary
-cache, Nix will build the package from source. So if <literal>nix-env
--i subversion</literal> results in Nix building stuff from source,
-then either the package is not built for your platform by the Nixpkgs
-build servers, or your version of Nixpkgs is too old or too new. For
-instance, if you have a very recent checkout of Nixpkgs, then the
-Nixpkgs build servers may not have had a chance to build everything
-and upload the resulting binaries to
-<uri>https://cache.nixos.org</uri>. The Nixpkgs channel is only
-updated after all binaries have been uploaded to the cache, so if you
-stick to the Nixpkgs channel (rather than using a Git checkout of the
-Nixpkgs tree), you will get binaries for most packages.</para></note>
-
-<para>Naturally, packages can also be uninstalled:
-
-<screen>
-$ nix-env -e subversion</screen>
-
-</para>
-
-<para>Upgrading to a new version is just as easy.  If you have a new
-release of Nix Packages, you can do:
-
-<screen>
-$ nix-env -u subversion</screen>
-
-This will <emphasis>only</emphasis> upgrade Subversion if there is a
-“newer” version in the new set of Nix expressions, as
-defined by some pretty arbitrary rules regarding ordering of version
-numbers (which generally do what you’d expect of them).  To just
-unconditionally replace Subversion with whatever version is in the Nix
-expressions, use <parameter>-i</parameter> instead of
-<parameter>-u</parameter>; <parameter>-i</parameter> will remove
-whatever version is already installed.</para>
-
-<para>You can also upgrade all packages for which there are newer
-versions:
-
-<screen>
-$ nix-env -u</screen>
-
-</para>
-
-<para>Sometimes it’s useful to be able to ask what
-<command>nix-env</command> would do, without actually doing it.  For
-instance, to find out what packages would be upgraded by
-<literal>nix-env -u</literal>, you can do
-
-<screen>
-$ nix-env -u --dry-run
-(dry run; not doing anything)
-upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
-upgrading `graphviz-1.10' to `graphviz-1.12'
-upgrading `coreutils-5.0' to `coreutils-5.2.1'</screen>
-
-</para>
-
-</chapter>
diff --git a/doc/manual/packages/binary-cache-substituter.xml b/doc/manual/packages/binary-cache-substituter.xml
deleted file mode 100644
index c6ceb9c80..000000000
--- a/doc/manual/packages/binary-cache-substituter.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink"
-         xmlns:xi="http://www.w3.org/2001/XInclude"
-         version="5.0"
-         xml:id="ssec-binary-cache-substituter">
-
-<title>Serving a Nix store via HTTP</title>
-
-<para>You can easily share the Nix store of a machine via HTTP. This
-allows other machines to fetch store paths from that machine to speed
-up installations. It uses the same <emphasis>binary cache</emphasis>
-mechanism that Nix usually uses to fetch pre-built binaries from
-<uri>https://cache.nixos.org</uri>.</para>
-
-<para>The daemon that handles binary cache requests via HTTP,
-<command>nix-serve</command>, is not part of the Nix distribution, but
-you can install it from Nixpkgs:
-
-<screen>
-$ nix-env -i nix-serve
-</screen>
-
-You can then start the server, listening for HTTP connections on
-whatever port you like:
-
-<screen>
-$ nix-serve -p 8080
-</screen>
-
-To check whether it works, try the following on the client:
-
-<screen>
-$ curl http://avalon:8080/nix-cache-info
-</screen>
-
-which should print something like:
-
-<screen>
-StoreDir: /nix/store
-WantMassQuery: 1
-Priority: 30
-</screen>
-
-</para>
-
-<para>On the client side, you can tell Nix to use your binary cache
-using <option>--option extra-binary-caches</option>, e.g.:
-
-<screen>
-$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
-</screen>
-
-The option <option>extra-binary-caches</option> tells Nix to use this
-binary cache in addition to your default caches, such as
-<uri>https://cache.nixos.org</uri>. Thus, for any path in the closure
-of Firefox, Nix will first check if the path is available on the
-server <literal>avalon</literal> or another binary caches. If not, it
-will fall back to building from source.</para>
-
-<para>You can also tell Nix to always use your binary cache by adding
-a line to the <filename linkend="sec-conf-file">nix.conf</filename>
-configuration file like this:
-
-<programlisting>
-binary-caches = http://avalon:8080/ https://cache.nixos.org/
-</programlisting>
-
-</para>
-
-</section>
diff --git a/doc/manual/packages/channels.xml b/doc/manual/packages/channels.xml
deleted file mode 100644
index 494a81966..000000000
--- a/doc/manual/packages/channels.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-channels">
-
-<title>Channels</title>
-
-<para>If you want to stay up to date with a set of packages, it’s not
-very convenient to manually download the latest set of Nix expressions
-for those packages and upgrade using <command>nix-env</command>.
-Fortunately, there’s a better way: <emphasis>Nix
-channels</emphasis>.</para>
-
-<para>A Nix channel is just a URL that points to a place that contains
-a set of Nix expressions and a manifest.  Using the command <link
-linkend="sec-nix-channel"><command>nix-channel</command></link> you
-can automatically stay up to date with whatever is available at that
-URL.</para>
-      
-<para>To see the list of official NixOS channels, visit <link
-xlink:href="https://nixos.org/channels" />.</para>
-
-<para>You can “subscribe” to a channel using
-<command>nix-channel --add</command>, e.g.,
-
-<screen>
-$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable</screen>
-
-subscribes you to a channel that always contains that latest version
-of the Nix Packages collection.  (Subscribing really just means that
-the URL is added to the file <filename>~/.nix-channels</filename>,
-where it is read by subsequent calls to <command>nix-channel
---update</command>.) You can “unsubscribe” using <command>nix-channel
---remove</command>:
-
-<screen>
-$ nix-channel --remove nixpkgs
-</screen>
-</para>
-
-<para>To obtain the latest Nix expressions available in a channel, do
-
-<screen>
-$ nix-channel --update</screen>
-
-This downloads and unpacks the Nix expressions in every channel
-(downloaded from <literal><emphasis>url</emphasis>/nixexprs.tar.bz2</literal>).
-It also makes the union of each channel’s Nix expressions available by
-default to <command>nix-env</command> operations (via the symlink
-<filename>~/.nix-defexpr/channels</filename>).  Consequently, you can
-then say
-
-<screen>
-$ nix-env -u</screen>
-
-to upgrade all packages in your profile to the latest versions
-available in the subscribed channels.</para>
-
-</chapter>
diff --git a/doc/manual/packages/copy-closure.xml b/doc/manual/packages/copy-closure.xml
deleted file mode 100644
index a336fb293..000000000
--- a/doc/manual/packages/copy-closure.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink"
-         xmlns:xi="http://www.w3.org/2001/XInclude"
-         version="5.0"
-         xml:id="ssec-copy-closure">
-
-<title>Copying Closures via SSH</title>
-
-<para>The command <command
-linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix
-store path along with all its dependencies to or from another machine
-via the SSH protocol.  It doesn’t copy store paths that are already
-present on the target machine.  For example, the following command
-copies Firefox with all its dependencies:
-
-<screen>
-$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen>
-
-See <xref linkend='sec-nix-copy-closure' /> for details.</para>
-
-<para>With <command linkend='refsec-nix-store-export'>nix-store
---export</command> and <command
-linkend='refsec-nix-store-import'>nix-store --import</command> you can
-write the closure of a store path (that is, the path and all its
-dependencies) to a file, and then unpack that file into another Nix
-store.  For example,
-
-<screen>
-$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure</screen>
-
-writes the closure of Firefox to a file.  You can then copy this file
-to another machine and install the closure:
-
-<screen>
-$ nix-store --import &lt; firefox.closure</screen>
-
-Any store paths in the closure that are already present in the target
-store are ignored.  It is also possible to pipe the export into
-another command, e.g. to copy and install a closure directly to/on
-another machine:
-
-<screen>
-$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \
-    ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen>
-
-However, <command>nix-copy-closure</command> is generally more
-efficient because it only copies paths that are not already present in
-the target Nix store.</para>
-
-</section>
diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml
deleted file mode 100644
index b506f22b0..000000000
--- a/doc/manual/packages/garbage-collection.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='sec-garbage-collection'>
-
-<title>Garbage Collection</title>
-
-<para><command>nix-env</command> operations such as upgrades
-(<option>-u</option>) and uninstall (<option>-e</option>) never
-actually delete packages from the system.  All they do (as shown
-above) is to create a new user environment that no longer contains
-symlinks to the “deleted” packages.</para>
-
-<para>Of course, since disk space is not infinite, unused packages
-should be removed at some point.  You can do this by running the Nix
-garbage collector.  It will remove from the Nix store any package
-not used (directly or indirectly) by any generation of any
-profile.</para>
-
-<para>Note however that as long as old generations reference a
-package, it will not be deleted.  After all, we wouldn’t be able to
-do a rollback otherwise.  So in order for garbage collection to be
-effective, you should also delete (some) old generations.  Of course,
-this should only be done if you are certain that you will not need to
-roll back.</para>
-
-<para>To delete all old (non-current) generations of your current
-profile:
-
-<screen>
-$ nix-env --delete-generations old</screen>
-
-Instead of <literal>old</literal> you can also specify a list of
-generations, e.g.,
-
-<screen>
-$ nix-env --delete-generations 10 11 14</screen>
-
-To delete all generations older than a specified number of days
-(except the current generation), use the <literal>d</literal>
-suffix. For example,
-
-<screen>
-$ nix-env --delete-generations 14d</screen>
-
-deletes all generations older than two weeks.</para>
-
-<para>After removing appropriate old generations you can run the
-garbage collector as follows:
-
-<screen>
-$ nix-store --gc</screen>
-
-The behaviour of the gargage collector is affected by the 
-<literal>keep-derivations</literal> (default: true) and <literal>keep-outputs</literal>
-(default: false) options in the Nix configuration file. The defaults will ensure
-that all derivations that are build-time dependencies of garbage collector roots
-will be kept and that all output paths that are runtime dependencies
-will be kept as well. All other derivations or paths will be collected. 
-(This is usually what you want, but while you are developing
-it may make sense to keep outputs to ensure that rebuild times are quick.)
-
-If you are feeling uncertain, you can also first view what files would
-be deleted:
-
-<screen>
-$ nix-store --gc --print-dead</screen>
-
-Likewise, the option <option>--print-live</option> will show the paths
-that <emphasis>won’t</emphasis> be deleted.</para>
-
-<para>There is also a convenient little utility
-<command>nix-collect-garbage</command>, which when invoked with the
-<option>-d</option> (<option>--delete-old</option>) switch deletes all
-old generations of all profiles in
-<filename>/nix/var/nix/profiles</filename>.  So
-
-<screen>
-$ nix-collect-garbage -d</screen>
-
-is a quick and easy way to clean up your system.</para>
-
-<xi:include href="garbage-collector-roots.xml" />
-
-</chapter>
diff --git a/doc/manual/packages/garbage-collector-roots.xml b/doc/manual/packages/garbage-collector-roots.xml
deleted file mode 100644
index 9c91862b0..000000000
--- a/doc/manual/packages/garbage-collector-roots.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-gc-roots">
-
-<title>Garbage Collector Roots</title>
-
-<para>The roots of the garbage collector are all store paths to which
-there are symlinks in the directory
-<filename><emphasis>prefix</emphasis>/nix/var/nix/gcroots</filename>.
-For instance, the following command makes the path
-<filename>/nix/store/d718ef...-foo</filename> a root of the collector:
-
-<screen>
-$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen>
-	
-That is, after this command, the garbage collector will not remove
-<filename>/nix/store/d718ef...-foo</filename> or any of its
-dependencies.</para>
-
-<para>Subdirectories of
-<filename><emphasis>prefix</emphasis>/nix/var/nix/gcroots</filename>
-are also searched for symlinks.  Symlinks to non-store paths are
-followed and searched for roots, but symlinks to non-store paths
-<emphasis>inside</emphasis> the paths reached in that way are not
-followed to prevent infinite recursion.</para>
-
-</section>
\ No newline at end of file
diff --git a/doc/manual/packages/package-management.xml b/doc/manual/packages/package-management.xml
deleted file mode 100644
index 61e55faeb..000000000
--- a/doc/manual/packages/package-management.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-<part xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id='chap-package-management'>
-
-<title>Package Management</title>
-
-<partintro>
-<para>This chapter discusses how to do package management with Nix,
-i.e., how to obtain, install, upgrade, and erase packages.  This is
-the “user’s” perspective of the Nix system — people
-who want to <emphasis>create</emphasis> packages should consult
-<xref linkend='chap-writing-nix-expressions' />.</para>
-</partintro>
-
-<xi:include href="basic-package-mgmt.xml" />
-<xi:include href="profiles.xml" />
-<xi:include href="garbage-collection.xml" />
-<xi:include href="channels.xml" />
-<xi:include href="sharing-packages.xml" />
-
-</part>
diff --git a/doc/manual/packages/profiles.xml b/doc/manual/packages/profiles.xml
deleted file mode 100644
index 3b8c5e293..000000000
--- a/doc/manual/packages/profiles.xml
+++ /dev/null
@@ -1,154 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-profiles">
-
-<title>Profiles</title>
-
-<para>Profiles and user environments are Nix’s mechanism for
-implementing the ability to allow different users to have different
-configurations, and to do atomic upgrades and rollbacks.  To
-understand how they work, it’s useful to know a bit about how Nix
-works.  In Nix, packages are stored in unique locations in the
-<emphasis>Nix store</emphasis> (typically,
-<filename>/nix/store</filename>).  For instance, a particular version
-of the Subversion package might be stored in a directory
-<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>,
-while another version might be stored in
-<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
-The long strings prefixed to the directory names are cryptographic
-hashes (to be precise, 160-bit truncations of SHA-256 hashes encoded
-in a base-32 notation) of <emphasis>all</emphasis> inputs involved in
-building the package — sources, dependencies, compiler flags, and so
-on.  So if two packages differ in any way, they end up in different
-locations in the file system, so they don’t interfere with each other.
-Here is what a part of a typical Nix store looks like:</para>
-
-<mediaobject>
-  <imageobject>
-    <imagedata fileref='../figures/user-environments.png' format='PNG' />
-  </imageobject>
-</mediaobject>
-
-<para>Of course, you wouldn’t want to type
-
-<screen>
-$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
-
-every time you want to run Subversion.  Of course we could set up the
-<literal>PATH</literal> environment variable to include the
-<filename>bin</filename> directory of every package we want to use,
-but this is not very convenient since changing <literal>PATH</literal>
-doesn’t take effect for already existing processes.  The solution Nix
-uses is to create directory trees of symlinks to
-<emphasis>activated</emphasis> packages.  These are called
-<emphasis>user environments</emphasis> and they are packages
-themselves (though automatically generated by
-<command>nix-env</command>), so they too reside in the Nix store.  For
-instance, in the figure above, the user environment
-<filename>/nix/store/0c1p5z4kda11...-user-env</filename> contains a
-symlink to just Subversion 1.1.2 (arrows in the figure indicate
-symlinks).  This would be what we would obtain if we had done
-
-<screen>
-$ nix-env -i subversion</screen>
-
-on a set of Nix expressions that contained Subversion 1.1.2.</para>
-
-<para>This doesn’t in itself solve the problem, of course; you
-wouldn’t want to type
-<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename>
-either.  That’s why there are symlinks outside of the store that point
-to the user environments in the store; for instance, the symlinks
-<filename>default-42-link</filename> and
-<filename>default-43-link</filename> in the example.  These are called
-<emphasis>generations</emphasis> since every time you perform a
-<command>nix-env</command> operation, a new user environment is
-generated based on the current one.  For instance, generation 43 was
-created from generation 42 when we did
-
-<screen>
-$ nix-env -i subversion firefox</screen>
-
-on a set of Nix expressions that contained Firefox and a new version
-of Subversion.</para>
-
-<para>Generations are grouped together into
-<emphasis>profiles</emphasis> so that different users don’t interfere
-with each other if they don’t want to.  For example:
-
-<screen>
-$ ls -l /nix/var/nix/profiles/
-...
-lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
-lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
-lrwxrwxrwx  1 eelco ... default -> default-43-link</screen>
-
-This shows a profile called <filename>default</filename>.  The file
-<filename>default</filename> itself is actually a symlink that points
-to the current generation.  When we do a <command>nix-env</command>
-operation, a new user environment and generation link are created
-based on the current one, and finally the <filename>default</filename>
-symlink is made to point at the new generation.  This last step is
-atomic on Unix, which explains how we can do atomic upgrades.  (Note
-that the building/installing of new packages doesn’t interfere in
-any way with old packages, since they are stored in different
-locations in the Nix store.)</para>
-
-<para>If you find that you want to undo a <command>nix-env</command>
-operation, you can just do
-
-<screen>
-$ nix-env --rollback</screen>
-
-which will just make the current generation link point at the previous
-link.  E.g., <filename>default</filename> would be made to point at
-<filename>default-42-link</filename>.  You can also switch to a
-specific generation:
-
-<screen>
-$ nix-env --switch-generation 43</screen>
-
-which in this example would roll forward to generation 43 again.  You
-can also see all available generations:
-
-<screen>
-$ nix-env --list-generations</screen></para>
-
-<para>You generally wouldn’t have
-<filename>/nix/var/nix/profiles/<emphasis>some-profile</emphasis>/bin</filename>
-in your <literal>PATH</literal>.  Rather, there is a symlink
-<filename>~/.nix-profile</filename> that points to your current
-profile.  This means that you should put
-<filename>~/.nix-profile/bin</filename> in your <literal>PATH</literal>
-(and indeed, that’s what the initialisation script
-<filename>/nix/etc/profile.d/nix.sh</filename> does).  This makes it
-easier to switch to a different profile.  You can do that using the
-command <command>nix-env --switch-profile</command>:
-
-<screen>
-$ nix-env --switch-profile /nix/var/nix/profiles/my-profile
-
-$ nix-env --switch-profile /nix/var/nix/profiles/default</screen>
-
-These commands switch to the <filename>my-profile</filename> and
-default profile, respectively.  If the profile doesn’t exist, it will
-be created automatically.  You should be careful about storing a
-profile in another location than the <filename>profiles</filename>
-directory, since otherwise it might not be used as a root of the
-garbage collector (see <xref linkend='sec-garbage-collection'
-/>).</para>
-
-<para>All <command>nix-env</command> operations work on the profile
-pointed to by <command>~/.nix-profile</command>, but you can override
-this using the <option>--profile</option> option (abbreviation
-<option>-p</option>):
-
-<screen>
-$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion</screen>
-
-This will <emphasis>not</emphasis> change the
-<command>~/.nix-profile</command> symlink.</para>
-
-</chapter>
diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
deleted file mode 100644
index aa5df2026..000000000
--- a/doc/manual/packages/s3-substituter.xml
+++ /dev/null
@@ -1,191 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<section xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink"
-         xmlns:xi="http://www.w3.org/2001/XInclude"
-         version="5.0"
-         xml:id="ssec-s3-substituter">
-
-<title>Serving a Nix store via S3</title>
-
-<para>Nix has built-in support for storing and fetching store paths
-from Amazon S3 and S3-compatible services. This uses the same
-<emphasis>binary</emphasis> cache mechanism that Nix usually uses to
-fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
-
-<para>The following options can be specified as URL parameters to
-the S3 URL:</para>
-
-<variablelist>
-  <varlistentry><term><literal>profile</literal></term>
-  <listitem>
-    <para>
-      The name of the AWS configuration profile to use. By default
-      Nix will use the <literal>default</literal> profile.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>region</literal></term>
-  <listitem>
-    <para>
-      The region of the S3 bucket. <literal>us–east-1</literal> by
-      default.
-    </para>
-
-    <para>
-      If your bucket is not in <literal>us–east-1</literal>, you
-      should always explicitly specify the region parameter.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>endpoint</literal></term>
-  <listitem>
-    <para>
-      The URL to your S3-compatible service, for when not using
-      Amazon S3. Do not specify this value if you're using Amazon
-      S3.
-    </para>
-    <note><para>This endpoint must support HTTPS and will use
-    path-based addressing instead of virtual host based
-    addressing.</para></note>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry><term><literal>scheme</literal></term>
-  <listitem>
-    <para>
-      The scheme used for S3 requests, <literal>https</literal>
-      (default) or <literal>http</literal>.  This option allows you to
-      disable HTTPS for binary caches which don't support it.
-    </para>
-    <note><para>HTTPS should be used if the cache might contain
-    sensitive information.</para></note>
-  </listitem>
-  </varlistentry>
-</variablelist>
-
-<para>In this example we will use the bucket named
-<literal>example-nix-cache</literal>.</para>
-
-<section xml:id="ssec-s3-substituter-anonymous-reads">
-  <title>Anonymous Reads to your S3-compatible binary cache</title>
-
-  <para>If your binary cache is publicly accessible and does not
-  require authentication, the simplest and easiest way to use Nix with
-  your S3 compatible binary cache is to use the HTTP URL for that
-  cache.</para>
-
-  <para>For AWS S3 the binary cache URL for example bucket will be
-  exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or
-  <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches,
-  consult that cache's documentation.</para>
-
-  <para>Your bucket will need the following bucket policy:</para>
-
-  <programlisting><![CDATA[
-{
-    "Id": "DirectReads",
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Sid": "AllowDirectReads",
-            "Action": [
-                "s3:GetObject",
-                "s3:GetBucketLocation"
-            ],
-            "Effect": "Allow",
-            "Resource": [
-                "arn:aws:s3:::example-nix-cache",
-                "arn:aws:s3:::example-nix-cache/*"
-            ],
-            "Principal": "*"
-        }
-    ]
-}
-]]></programlisting>
-</section>
-
-<section xml:id="ssec-s3-substituter-authenticated-reads">
-  <title>Authenticated Reads to your S3 binary cache</title>
-
-  <para>For AWS S3 the binary cache URL for example bucket will be
-  exactly <uri>s3://example-nix-cache</uri>.</para>
-
-  <para>Nix will use the <link
-  xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default
-  credential provider chain</link> for authenticating requests to
-  Amazon S3.</para>
-
-  <para>Nix supports authenticated reads from Amazon S3 and S3
-  compatible binary caches.</para>
-
-  <para>Your bucket will need a bucket policy allowing the desired
-  users to perform the <literal>s3:GetObject</literal> and
-  <literal>s3:GetBucketLocation</literal> action on all objects in the
-  bucket. The anonymous policy in <xref
-  linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to
-  have a restricted <literal>Principal</literal> to support
-  this.</para>
-</section>
-
-
-<section xml:id="ssec-s3-substituter-authenticated-writes">
-  <title>Authenticated Writes to your S3-compatible binary cache</title>
-
-  <para>Nix support fully supports writing to Amazon S3 and S3
-  compatible buckets. The binary cache URL for our example bucket will
-  be <uri>s3://example-nix-cache</uri>.</para>
-
-  <para>Nix will use the <link
-  xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default
-  credential provider chain</link> for authenticating requests to
-  Amazon S3.</para>
-
-  <para>Your account will need the following IAM policy to
-  upload to the cache:</para>
-
-  <programlisting><![CDATA[
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Sid": "UploadToCache",
-      "Effect": "Allow",
-      "Action": [
-        "s3:AbortMultipartUpload",
-        "s3:GetBucketLocation",
-        "s3:GetObject",
-        "s3:ListBucket",
-        "s3:ListBucketMultipartUploads",
-        "s3:ListMultipartUploadParts",
-        "s3:PutObject"
-      ],
-      "Resource": [
-        "arn:aws:s3:::example-nix-cache",
-        "arn:aws:s3:::example-nix-cache/*"
-      ]
-    }
-  ]
-}
-]]></programlisting>
-
-</section>
-
-<section><title>Examples</title>
-
-<para>To upload with a specific credential profile for Amazon S3:</para>
-
-<screen>
-nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;region=eu-west-2' nixpkgs.hello
-</screen>
-
-<para>To upload to an S3-compatible binary cache:</para>
-
-<screen>
-nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello
-</screen>
-
-</section>
-
-</section>
diff --git a/doc/manual/packages/sharing-packages.xml b/doc/manual/packages/sharing-packages.xml
deleted file mode 100644
index bb6c52b8f..000000000
--- a/doc/manual/packages/sharing-packages.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-sharing-packages">
-
-<title>Sharing Packages Between Machines</title>
-
-<para>Sometimes you want to copy a package from one machine to
-another.  Or, you want to install some packages and you know that
-another machine already has some or all of those packages or their
-dependencies.  In that case there are mechanisms to quickly copy
-packages between machines.</para>
-
-<xi:include href="binary-cache-substituter.xml" />
-<xi:include href="copy-closure.xml" />
-<xi:include href="ssh-substituter.xml" />
-<xi:include href="s3-substituter.xml" />
-
-</chapter>
diff --git a/doc/manual/packages/ssh-substituter.xml b/doc/manual/packages/ssh-substituter.xml
deleted file mode 100644
index 8db3f9662..000000000
--- a/doc/manual/packages/ssh-substituter.xml
+++ /dev/null
@@ -1,73 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink"
-         xmlns:xi="http://www.w3.org/2001/XInclude"
-         version="5.0"
-         xml:id="ssec-ssh-substituter">
-
-<title>Serving a Nix store via SSH</title>
-
-<para>You can tell Nix to automatically fetch needed binaries from a
-remote Nix store via SSH. For example, the following installs Firefox,
-automatically fetching any store paths in Firefox’s closure if they
-are available on the server <literal>avalon</literal>:
-
-<screen>
-$ nix-env -i firefox --substituters ssh://alice@avalon
-</screen>
-
-This works similar to the binary cache substituter that Nix usually
-uses, only using SSH instead of HTTP: if a store path
-<literal>P</literal> is needed, Nix will first check if it’s available
-in the Nix store on <literal>avalon</literal>. If not, it will fall
-back to using the binary cache substituter, and then to building from
-source.</para>
-
-<note><para>The SSH substituter currently does not allow you to enter
-an SSH passphrase interactively. Therefore, you should use
-<command>ssh-add</command> to load the decrypted private key into
-<command>ssh-agent</command>.</para></note>
-
-<para>You can also copy the closure of some store path, without
-installing it into your profile, e.g.
-
-<screen>
-$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon
-</screen>
-
-This is essentially equivalent to doing
-
-<screen>
-$ nix-copy-closure --from alice@avalon /nix/store/m85bxg…-firefox-34.0.5
-</screen>
-
-</para>
-
-<para>You can use SSH’s <emphasis>forced command</emphasis> feature to
-set up a restricted user account for SSH substituter access, allowing
-read-only access to the local Nix store, but nothing more. For
-example, add the following lines to <filename>sshd_config</filename>
-to restrict the user <literal>nix-ssh</literal>:
-
-<programlisting>
-Match User nix-ssh
-  AllowAgentForwarding no
-  AllowTcpForwarding no
-  PermitTTY no
-  PermitTunnel no
-  X11Forwarding no
-  ForceCommand nix-store --serve
-Match All
-</programlisting>
-
-On NixOS, you can accomplish the same by adding the following to your
-<filename>configuration.nix</filename>:
-
-<programlisting>
-nix.sshServe.enable = true;
-nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
-</programlisting>
-
-where the latter line lists the public keys of users that are allowed
-to connect.</para>
-
-</section>
diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
deleted file mode 100644
index 2655d68e3..000000000
--- a/doc/manual/release-notes/release-notes.xml
+++ /dev/null
@@ -1,51 +0,0 @@
-<appendix xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-relnotes">
-
-<title>Nix Release Notes</title>
-
-<!--
-<partintro>
-<para>This section lists the release notes for each stable version of Nix.</para>
-</partintro>
--->
-
-<xi:include href="rl-2.3.xml" />
-<xi:include href="rl-2.2.xml" />
-<xi:include href="rl-2.1.xml" />
-<xi:include href="rl-2.0.xml" />
-<xi:include href="rl-1.11.10.xml" />
-<xi:include href="rl-1.11.xml" />
-<xi:include href="rl-1.10.xml" />
-<xi:include href="rl-1.9.xml" />
-<xi:include href="rl-1.8.xml" />
-<xi:include href="rl-1.7.xml" />
-<xi:include href="rl-1.6.1.xml" />
-<xi:include href="rl-1.6.xml" />
-<xi:include href="rl-1.5.2.xml" />
-<xi:include href="rl-1.5.xml" />
-<xi:include href="rl-1.4.xml" />
-<xi:include href="rl-1.3.xml" />
-<xi:include href="rl-1.2.xml" />
-<xi:include href="rl-1.1.xml" />
-<xi:include href="rl-1.0.xml" />
-<xi:include href="rl-0.16.xml" />
-<xi:include href="rl-0.15.xml" />
-<xi:include href="rl-0.14.xml" />
-<xi:include href="rl-0.13.xml" />
-<xi:include href="rl-0.12.xml" />
-<xi:include href="rl-0.11.xml" />
-<xi:include href="rl-0.10.1.xml" />
-<xi:include href="rl-0.10.xml" />
-<xi:include href="rl-0.9.2.xml" />
-<xi:include href="rl-0.9.1.xml" />
-<xi:include href="rl-0.9.xml" />
-<xi:include href="rl-0.8.1.xml" />
-<xi:include href="rl-0.8.xml" />
-<xi:include href="rl-0.7.xml" />
-<xi:include href="rl-0.6.xml" />
-<xi:include href="rl-0.5.xml" />
-
-</appendix>
diff --git a/doc/manual/release-notes/rl-0.10.1.xml b/doc/manual/release-notes/rl-0.10.1.xml
deleted file mode 100644
index 95829323d..000000000
--- a/doc/manual/release-notes/rl-0.10.1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.10.1">
-
-<title>Release 0.10.1 (2006-10-11)</title>
-
-<para>This release fixes two somewhat obscure bugs that occur when
-evaluating Nix expressions that are stored inside the Nix store
-(<literal>NIX-67</literal>).  These do not affect most users.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.10.xml b/doc/manual/release-notes/rl-0.10.xml
deleted file mode 100644
index 01f70acfd..000000000
--- a/doc/manual/release-notes/rl-0.10.xml
+++ /dev/null
@@ -1,323 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.10">
-
-<title>Release 0.10 (2006-10-06)</title>
-
-<note><para>This version of Nix uses Berkeley DB 4.4 instead of 4.3.
-The database is upgraded automatically, but you should be careful not
-to use old versions of Nix that still use Berkeley DB 4.3.  In
-particular, if you use a Nix installed through Nix, you should run
-
-<screen>
-$ nix-store --clear-substitutes</screen>
-
-first.</para></note>
-
-<warning><para>Also, the database schema has changed slighted to fix a
-performance issue (see below).  When you run any Nix 0.10 command for
-the first time, the database will be upgraded automatically.  This is
-irreversible.</para></warning>
-
-<itemizedlist>
-
-
-  <!-- Usability / features -->
-
-
-  <listitem><para><command>nix-env</command> usability improvements:
-
-    <itemizedlist>
-
-      <listitem><para>An option <option>--compare-versions</option>
-      (or <option>-c</option>) has been added to <command>nix-env
-      --query</command> to allow you to compare installed versions of
-      packages to available versions, or vice versa.  An easy way to
-      see if you are up to date with what’s in your subscribed
-      channels is <literal>nix-env -qc \*</literal>.</para></listitem>
-
-      <listitem><para><literal>nix-env --query</literal> now takes as
-      arguments a list of package names about which to show
-      information, just like <option>--install</option>, etc.: for
-      example, <literal>nix-env -q gcc</literal>.  Note that to show
-      all derivations, you need to specify
-      <literal>\*</literal>.</para></listitem>
-
-      <listitem><para><literal>nix-env -i
-      <emphasis>pkgname</emphasis></literal> will now install
-      the highest available version of
-      <emphasis>pkgname</emphasis>, rather than installing all
-      available versions (which would probably give collisions)
-      (<literal>NIX-31</literal>).</para></listitem>
-
-      <listitem><para><literal>nix-env (-i|-u) --dry-run</literal> now
-      shows exactly which missing paths will be built or
-      substituted.</para></listitem>
-
-      <listitem><para><literal>nix-env -qa --description</literal>
-      shows human-readable descriptions of packages, provided that
-      they have a <literal>meta.description</literal> attribute (which
-      most packages in Nixpkgs don’t have yet).</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-
-  <listitem><para>New language features:
-
-    <itemizedlist>
-
-      <listitem><para>Reference scanning (which happens after each
-      build) is much faster and takes a constant amount of
-      memory.</para></listitem>
-
-      <listitem><para>String interpolation.  Expressions like
-
-<programlisting>
-"--with-freetype2-library=" + freetype + "/lib"</programlisting>
-
-      can now be written as
-
-<programlisting>
-"--with-freetype2-library=${freetype}/lib"</programlisting>
-
-      You can write arbitrary expressions within
-      <literal>${<emphasis>...</emphasis>}</literal>, not just
-      identifiers.</para></listitem>
-
-      <listitem><para>Multi-line string literals.</para></listitem>
-
-      <listitem><para>String concatenations can now involve
-      derivations, as in the example <code>"--with-freetype2-library="
-      + freetype + "/lib"</code>.  This was not previously possible
-      because we need to register that a derivation that uses such a
-      string is dependent on <literal>freetype</literal>.  The
-      evaluator now properly propagates this information.
-      Consequently, the subpath operator (<literal>~</literal>) has
-      been deprecated.</para></listitem>
-
-      <listitem><para>Default values of function arguments can now
-      refer to other function arguments; that is, all arguments are in
-      scope in the default values
-      (<literal>NIX-45</literal>).</para></listitem>
-
-      <!--
-      <listitem><para>TODO: domain checks (r5895).</para></listitem>
-      -->
-
-      <listitem><para>Lots of new built-in primitives, such as
-      functions for list manipulation and integer arithmetic.  See the
-      manual for a complete list.  All primops are now available in
-      the set <varname>builtins</varname>, allowing one to test for
-      the availability of primop in a backwards-compatible
-      way.</para></listitem>
-
-      <listitem><para>Real let-expressions: <literal>let x = ...;
-      ... z = ...; in ...</literal>.</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-
-  <listitem><para>New commands <command>nix-pack-closure</command> and
-  <command>nix-unpack-closure</command> than can be used to easily
-  transfer a store path with all its dependencies to another machine.
-  Very convenient whenever you have some package on your machine and
-  you want to copy it somewhere else.</para></listitem>
-
-
-  <listitem><para>XML support:
-
-    <itemizedlist>
-
-      <listitem><para><literal>nix-env -q --xml</literal> prints the
-      installed or available packages in an XML representation for
-      easy processing by other tools.</para></listitem>
-
-      <listitem><para><literal>nix-instantiate --eval-only
-      --xml</literal> prints an XML representation of the resulting
-      term.  (The new flag <option>--strict</option> forces ‘deep’
-      evaluation of the result, i.e., list elements and attributes are
-      evaluated recursively.)</para></listitem>
-
-      <listitem><para>In Nix expressions, the primop
-      <function>builtins.toXML</function> converts a term to an XML
-      representation.  This is primarily useful for passing structured
-      information to builders.</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-
-  <listitem><para>You can now unambiguously specify which derivation to
-  build or install in <command>nix-env</command>,
-  <command>nix-instantiate</command> and <command>nix-build</command>
-  using the <option>--attr</option> / <option>-A</option> flags, which
-  takes an attribute name as argument.  (Unlike symbolic package names
-  such as <literal>subversion-1.4.0</literal>, attribute names in an
-  attribute set are unique.)  For instance, a quick way to perform a
-  test build of a package in Nixpkgs is <literal>nix-build
-  pkgs/top-level/all-packages.nix -A
-  <emphasis>foo</emphasis></literal>.  <literal>nix-env -q
-  --attr</literal> shows the attribute names corresponding to each
-  derivation.</para></listitem>
-
-
-  <listitem><para>If the top-level Nix expression used by
-  <command>nix-env</command>, <command>nix-instantiate</command> or
-  <command>nix-build</command> evaluates to a function whose arguments
-  all have default values, the function will be called automatically.
-  Also, the new command-line switch <option>--arg
-  <emphasis>name</emphasis>
-  <emphasis>value</emphasis></option> can be used to specify
-  function arguments on the command line.</para></listitem>
-
-
-  <listitem><para><literal>nix-install-package --url
-  <emphasis>URL</emphasis></literal> allows a package to be
-  installed directly from the given URL.</para></listitem>
-
-
-  <listitem><para>Nix now works behind an HTTP proxy server; just set
-  the standard environment variables <literal>http_proxy</literal>,
-  <literal>https_proxy</literal>, <literal>ftp_proxy</literal> or
-  <literal>all_proxy</literal> appropriately.  Functions such as
-  <function>fetchurl</function> in Nixpkgs also respect these
-  variables.</para></listitem>
-
-
-  <listitem><para><literal>nix-build -o
-  <emphasis>symlink</emphasis></literal> allows the symlink to
-  the build result to be named something other than
-  <literal>result</literal>.</para></listitem>
-
-
-  <!-- Stability / performance / etc. -->
-
-
-  <listitem><para>Platform support:
-
-    <itemizedlist>
-
-      <listitem><para>Support for 64-bit platforms, provided a <link
-      xlink:href="http://bugzilla.sen.cwi.nl:8080/show_bug.cgi?id=606">suitably
-      patched ATerm library</link> is used.  Also, files larger than 2
-      GiB are now supported.</para></listitem>
-
-      <listitem><para>Added support for Cygwin (Windows,
-      <literal>i686-cygwin</literal>), Mac OS X on Intel
-      (<literal>i686-darwin</literal>) and Linux on PowerPC
-      (<literal>powerpc-linux</literal>).</para></listitem>
-
-      <listitem><para>Users of SMP and multicore machines will
-      appreciate that the number of builds to be performed in parallel
-      can now be specified in the configuration file in the
-      <literal>build-max-jobs</literal> setting.</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-
-  <listitem><para>Garbage collector improvements:
-
-    <itemizedlist>
-
-      <listitem><para>Open files (such as running programs) are now
-      used as roots of the garbage collector.  This prevents programs
-      that have been uninstalled from being garbage collected while
-      they are still running.  The script that detects these
-      additional runtime roots
-      (<filename>find-runtime-roots.pl</filename>) is inherently
-      system-specific, but it should work on Linux and on all
-      platforms that have the <command>lsof</command>
-      utility.</para></listitem>
-
-      <listitem><para><literal>nix-store --gc</literal>
-      (a.k.a. <command>nix-collect-garbage</command>) prints out the
-      number of bytes freed on standard output.  <literal>nix-store
-      --gc --print-dead</literal> shows how many bytes would be freed
-      by an actual garbage collection.</para></listitem>
-
-      <listitem><para><literal>nix-collect-garbage -d</literal>
-      removes all old generations of <emphasis>all</emphasis> profiles
-      before calling the actual garbage collector (<literal>nix-store
-      --gc</literal>).  This is an easy way to get rid of all old
-      packages in the Nix store.</para></listitem>
-
-      <listitem><para><command>nix-store</command> now has an
-      operation <option>--delete</option> to delete specific paths
-      from the Nix store.  It won’t delete reachable (non-garbage)
-      paths unless <option>--ignore-liveness</option> is
-      specified.</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-
-  <listitem><para>Berkeley DB 4.4’s process registry feature is used
-  to recover from crashed Nix processes.</para></listitem>
-
-  <!--  <listitem><para>TODO: shared stores.</para></listitem> -->
-
-  <listitem><para>A performance issue has been fixed with the
-  <literal>referer</literal> table, which stores the inverse of the
-  <literal>references</literal> table (i.e., it tells you what store
-  paths refer to a given path).  Maintaining this table could take a
-  quadratic amount of time, as well as a quadratic amount of Berkeley
-  DB log file space (in particular when running the garbage collector)
-  (<literal>NIX-23</literal>).</para></listitem>
-
-  <listitem><para>Nix now catches the <literal>TERM</literal> and
-  <literal>HUP</literal> signals in addition to the
-  <literal>INT</literal> signal.  So you can now do a <literal>killall
-  nix-store</literal> without triggering a database
-  recovery.</para></listitem>
-
-  <listitem><para><command>bsdiff</command> updated to version
-  4.3.</para></listitem>
-
-  <listitem><para>Substantial performance improvements in expression
-  evaluation and <literal>nix-env -qa</literal>, all thanks to <link
-  xlink:href="http://valgrind.org/">Valgrind</link>.  Memory use has
-  been reduced by a factor 8 or so.  Big speedup by memoisation of
-  path hashing.</para></listitem>
-
-  <listitem><para>Lots of bug fixes, notably:
-
-    <itemizedlist>
-
-      <listitem><para>Make sure that the garbage collector can run
-      successfully when the disk is full
-      (<literal>NIX-18</literal>).</para></listitem>
-
-      <listitem><para><command>nix-env</command> now locks the profile
-      to prevent races between concurrent <command>nix-env</command>
-      operations on the same profile
-      (<literal>NIX-7</literal>).</para></listitem>
-
-      <listitem><para>Removed misleading messages from
-      <literal>nix-env -i</literal> (e.g., <literal>installing
-      `foo'</literal> followed by <literal>uninstalling
-      `foo'</literal>) (<literal>NIX-17</literal>).</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-  <listitem><para>Nix source distributions are a lot smaller now since
-  we no longer include a full copy of the Berkeley DB source
-  distribution (but only the bits we need).</para></listitem>
-
-  <listitem><para>Header files are now installed so that external
-  programs can use the Nix libraries.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.11.xml b/doc/manual/release-notes/rl-0.11.xml
deleted file mode 100644
index 7ad0ab5b7..000000000
--- a/doc/manual/release-notes/rl-0.11.xml
+++ /dev/null
@@ -1,261 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-0.11">
-
-<title>Release 0.11 (2007-12-31)</title>
-
-<para>Nix 0.11 has many improvements over the previous stable release.
-The most important improvement is secure multi-user support.  It also
-features many usability enhancements and language extensions, many of
-them prompted by NixOS, the purely functional Linux distribution based
-on Nix.  Here is an (incomplete) list:</para>
-
-
-<itemizedlist>
-
-
-  <listitem><para>Secure multi-user support.  A single Nix store can
-  now be shared between multiple (possible untrusted) users.  This is
-  an important feature for NixOS, where it allows non-root users to
-  install software.  The old setuid method for sharing a store between
-  multiple users has been removed.  Details for setting up a
-  multi-user store can be found in the manual.</para></listitem>
-
-
-  <listitem><para>The new command <command>nix-copy-closure</command>
-  gives you an easy and efficient way to exchange software between
-  machines.  It copies the missing parts of the closure of a set of
-  store path to or from a remote machine via
-  <command>ssh</command>.</para></listitem>
-
-
-  <listitem><para>A new kind of string literal: strings between double
-  single-quotes (<literal>''</literal>) have indentation
-  “intelligently” removed.  This allows large strings (such as shell
-  scripts or configuration file fragments in NixOS) to cleanly follow
-  the indentation of the surrounding expression.  It also requires
-  much less escaping, since <literal>''</literal> is less common in
-  most languages than <literal>"</literal>.</para></listitem>
-
-
-  <listitem><para><command>nix-env</command> <option>--set</option>
-  modifies the current generation of a profile so that it contains
-  exactly the specified derivation, and nothing else.  For example,
-  <literal>nix-env -p /nix/var/nix/profiles/browser --set
-  firefox</literal> lets the profile named
-  <filename>browser</filename> contain just Firefox.</para></listitem>
-
-
-  <listitem><para><command>nix-env</command> now maintains
-  meta-information about installed packages in profiles.  The
-  meta-information is the contents of the <varname>meta</varname>
-  attribute of derivations, such as <varname>description</varname> or
-  <varname>homepage</varname>.  The command <literal>nix-env -q --xml
-  --meta</literal> shows all meta-information.</para></listitem>
-
-
-  <listitem><para><command>nix-env</command> now uses the
-  <varname>meta.priority</varname> attribute of derivations to resolve
-  filename collisions between packages.  Lower priority values denote
-  a higher priority.  For instance, the GCC wrapper package and the
-  Binutils package in Nixpkgs both have a file
-  <filename>bin/ld</filename>, so previously if you tried to install
-  both you would get a collision.  Now, on the other hand, the GCC
-  wrapper declares a higher priority than Binutils, so the former’s
-  <filename>bin/ld</filename> is symlinked in the user
-  environment.</para></listitem>
-
-
-  <listitem><para><command>nix-env -i / -u</command>: instead of
-  breaking package ties by version, break them by priority and version
-  number.  That is, if there are multiple packages with the same name,
-  then pick the package with the highest priority, and only use the
-  version if there are multiple packages with the same
-  priority.</para>
-
-  <para>This makes it possible to mark specific versions/variant in
-  Nixpkgs more or less desirable than others.  A typical example would
-  be a beta version of some package (e.g.,
-  <literal>gcc-4.2.0rc1</literal>) which should not be installed even
-  though it is the highest version, except when it is explicitly
-  selected (e.g., <literal>nix-env -i
-  gcc-4.2.0rc1</literal>).</para></listitem>
-
-
-  <listitem><para><command>nix-env --set-flag</command> allows meta
-  attributes of installed packages to be modified.  There are several
-  attributes that can be usefully modified, because they affect the
-  behaviour of <command>nix-env</command> or the user environment
-  build script:
-
-    <itemizedlist>
-
-      <listitem><para><varname>meta.priority</varname> can be changed
-      to resolve filename clashes (see above).</para></listitem>
-
-      <listitem><para><varname>meta.keep</varname> can be set to
-      <literal>true</literal> to prevent the package from being
-      upgraded or replaced.  Useful if you want to hang on to an older
-      version of a package.</para></listitem>
-
-      <listitem><para><varname>meta.active</varname> can be set to
-      <literal>false</literal> to “disable” the package.  That is, no
-      symlinks will be generated to the files of the package, but it
-      remains part of the profile (so it won’t be garbage-collected).
-      Set it back to <literal>true</literal> to re-enable the
-      package.</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-
-  <listitem><para><command>nix-env -q</command> now has a flag
-  <option>--prebuilt-only</option> (<option>-b</option>) that causes
-  <command>nix-env</command> to show only those derivations whose
-  output is already in the Nix store or that can be substituted (i.e.,
-  downloaded from somewhere).  In other words, it shows the packages
-  that can be installed “quickly”, i.e., don’t need to be built from
-  source.  The <option>-b</option> flag is also available in
-  <command>nix-env -i</command> and <command>nix-env -u</command> to
-  filter out derivations for which no pre-built binary is
-  available.</para></listitem>
-
-
-  <listitem><para>The new option <option>--argstr</option> (in
-  <command>nix-env</command>, <command>nix-instantiate</command> and
-  <command>nix-build</command>) is like <option>--arg</option>, except
-  that the value is a string.  For example, <literal>--argstr system
-  i686-linux</literal> is equivalent to <literal>--arg system
-  \"i686-linux\"</literal> (note that <option>--argstr</option>
-  prevents annoying quoting around shell arguments).</para></listitem>
-
-
-  <listitem><para><command>nix-store</command> has a new operation
-  <option>--read-log</option> (<option>-l</option>)
-  <parameter>paths</parameter> that shows the build log of the given
-  paths.</para></listitem>
-
-
-  <!--
-  <listitem><para>TODO: semantic cleanups of string concatenation
-  etc. (mostly in r6740).</para></listitem>
-  -->
-
-
-  <listitem><para>Nix now uses Berkeley DB 4.5.  The database is
-  upgraded automatically, but you should be careful not to use old
-  versions of Nix that still use Berkeley DB 4.4.</para></listitem>
-
-
-  <!-- foo
-  <listitem><para>TODO: option <option>- -reregister</option> in
-  <command>nix-store - -register-validity</command>.</para></listitem>
-  -->
-
-
-  <listitem><para>The option <option>--max-silent-time</option>
-  (corresponding to the configuration setting
-  <literal>build-max-silent-time</literal>) allows you to set a
-  timeout on builds — if a build produces no output on
-  <literal>stdout</literal> or <literal>stderr</literal> for the given
-  number of seconds, it is terminated.  This is useful for recovering
-  automatically from builds that are stuck in an infinite
-  loop.</para></listitem>
-
-
-  <listitem><para><command>nix-channel</command>: each subscribed
-  channel is its own attribute in the top-level expression generated
-  for the channel.  This allows disambiguation (e.g. <literal>nix-env
-  -i -A nixpkgs_unstable.firefox</literal>).</para></listitem>
-
-
-  <listitem><para>The substitutes table has been removed from the
-  database.  This makes operations such as <command>nix-pull</command>
-  and <command>nix-channel --update</command> much, much
-  faster.</para></listitem>
-
-
-  <listitem><para><command>nix-pull</command> now supports
-  bzip2-compressed manifests.  This speeds up
-  channels.</para></listitem>
-
-
-  <listitem><para><command>nix-prefetch-url</command> now has a
-  limited form of caching.  This is used by
-  <command>nix-channel</command> to prevent unnecessary downloads when
-  the channel hasn’t changed.</para></listitem>
-
-
-  <listitem><para><command>nix-prefetch-url</command> now by default
-  computes the SHA-256 hash of the file instead of the MD5 hash.  In
-  calls to <function>fetchurl</function> you should pass the
-  <literal>sha256</literal> attribute instead of
-  <literal>md5</literal>.  You can pass either a hexadecimal or a
-  base-32 encoding of the hash.</para></listitem>
-
-
-  <listitem><para>Nix can now perform builds in an automatically
-  generated “chroot”.  This prevents a builder from accessing stuff
-  outside of the Nix store, and thus helps ensure purity.  This is an
-  experimental feature.</para></listitem>
-
-
-  <listitem><para>The new command <command>nix-store
-  --optimise</command> reduces Nix store disk space usage by finding
-  identical files in the store and hard-linking them to each other.
-  It typically reduces the size of the store by something like
-  25-35%.</para></listitem>
-
-
-  <listitem><para><filename>~/.nix-defexpr</filename> can now be a
-  directory, in which case the Nix expressions in that directory are
-  combined into an attribute set, with the file names used as the
-  names of the attributes.  The command <command>nix-env
-  --import</command> (which set the
-  <filename>~/.nix-defexpr</filename> symlink) is
-  removed.</para></listitem>
-
-
-  <listitem><para>Derivations can specify the new special attribute
-  <varname>allowedReferences</varname> to enforce that the references
-  in the output of a derivation are a subset of a declared set of
-  paths.  For example, if <varname>allowedReferences</varname> is an
-  empty list, then the output must not have any references.  This is
-  used in NixOS to check that generated files such as initial ramdisks
-  for booting Linux don’t have any dependencies.</para></listitem>
-
-
-  <listitem><para>The new attribute
-  <varname>exportReferencesGraph</varname> allows builders access to
-  the references graph of their inputs.  This is used in NixOS for
-  tasks such as generating ISO-9660 images that contain a Nix store
-  populated with the closure of certain paths.</para></listitem>
-
-
-  <listitem><para>Fixed-output derivations (like
-  <function>fetchurl</function>) can define the attribute
-  <varname>impureEnvVars</varname> to allow external environment
-  variables to be passed to builders.  This is used in Nixpkgs to
-  support proxy configuration, among other things.</para></listitem>
-
-
-  <listitem><para>Several new built-in functions:
-  <function>builtins.attrNames</function>,
-  <function>builtins.filterSource</function>,
-  <function>builtins.isAttrs</function>,
-  <function>builtins.isFunction</function>,
-  <function>builtins.listToAttrs</function>,
-  <function>builtins.stringLength</function>,
-  <function>builtins.sub</function>,
-  <function>builtins.substring</function>,
-  <function>throw</function>,
-  <function>builtins.trace</function>,
-  <function>builtins.readFile</function>.</para></listitem>
-
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.12.xml b/doc/manual/release-notes/rl-0.12.xml
deleted file mode 100644
index c30124786..000000000
--- a/doc/manual/release-notes/rl-0.12.xml
+++ /dev/null
@@ -1,175 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-0.12">
-
-<title>Release 0.12 (2008-11-20)</title>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Nix no longer uses Berkeley DB to store Nix store metadata.
-    The principal advantages of the new storage scheme are: it works
-    properly over decent implementations of NFS (allowing Nix stores
-    to be shared between multiple machines); no recovery is needed
-    when a Nix process crashes; no write access is needed for
-    read-only operations; no more running out of Berkeley DB locks on
-    certain operations.</para>
-
-    <para>You still need to compile Nix with Berkeley DB support if
-    you want Nix to automatically convert your old Nix store to the
-    new schema.  If you don’t need this, you can build Nix with the
-    <filename>configure</filename> option
-    <option>--disable-old-db-compat</option>.</para>
-
-    <para>After the automatic conversion to the new schema, you can
-    delete the old Berkeley DB files:
-
-    <screen>
-$ cd /nix/var/nix/db
-$ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</screen>
-
-    The new metadata is stored in the directories
-    <filename>/nix/var/nix/db/info</filename> and
-    <filename>/nix/var/nix/db/referrer</filename>.  Though the
-    metadata is stored in human-readable plain-text files, they are
-    not intended to be human-editable, as Nix is rather strict about
-    the format.</para>
-
-    <para>The new storage schema may or may not require less disk
-    space than the Berkeley DB environment, mostly depending on the
-    cluster size of your file system.  With 1 KiB clusters (which
-    seems to be the <literal>ext3</literal> default nowadays) it
-    usually takes up much less space.</para>
-  </listitem>
-
-  <listitem><para>There is a new substituter that copies paths
-  directly from other (remote) Nix stores mounted somewhere in the
-  filesystem.  For instance, you can speed up an installation by
-  mounting some remote Nix store that already has the packages in
-  question via NFS or <literal>sshfs</literal>.  The environment
-  variable <literal>NIX_OTHER_STORES</literal> specifies the locations of
-  the remote Nix directories,
-  e.g. <literal>/mnt/remote-fs/nix</literal>.</para></listitem>
-
-  <listitem><para>New <command>nix-store</command> operations
-  <option>--dump-db</option> and <option>--load-db</option> to dump
-  and reload the Nix database.</para></listitem>
-
-  <listitem><para>The garbage collector has a number of new options to
-  allow only some of the garbage to be deleted.  The option
-  <option>--max-freed <emphasis>N</emphasis></option> tells the
-  collector to stop after at least <emphasis>N</emphasis> bytes
-  have been deleted.  The option <option>--max-links
-  <emphasis>N</emphasis></option> tells it to stop after the
-  link count on <filename>/nix/store</filename> has dropped below
-  <emphasis>N</emphasis>.  This is useful for very large Nix
-  stores on filesystems with a 32000 subdirectories limit (like
-  <literal>ext3</literal>).  The option <option>--use-atime</option>
-  causes store paths to be deleted in order of ascending last access
-  time.  This allows non-recently used stuff to be deleted.  The
-  option <option>--max-atime <emphasis>time</emphasis></option>
-  specifies an upper limit to the last accessed time of paths that may
-  be deleted.  For instance,
-
-    <screen>
-    $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
-
-  deletes everything that hasn’t been accessed in two months.</para></listitem>
-
-  <listitem><para><command>nix-env</command> now uses optimistic
-  profile locking when performing an operation like installing or
-  upgrading, instead of setting an exclusive lock on the profile.
-  This allows multiple <command>nix-env -i / -u / -e</command>
-  operations on the same profile in parallel.  If a
-  <command>nix-env</command> operation sees at the end that the profile
-  was changed in the meantime by another process, it will just
-  restart.  This is generally cheap because the build results are
-  still in the Nix store.</para></listitem>
-
-  <listitem><para>The option <option>--dry-run</option> is now
-  supported by <command>nix-store -r</command> and
-  <command>nix-build</command>.</para></listitem>
-
-  <listitem><para>The information previously shown by
-  <option>--dry-run</option> (i.e., which derivations will be built
-  and which paths will be substituted) is now always shown by
-  <command>nix-env</command>, <command>nix-store -r</command> and
-  <command>nix-build</command>.  The total download size of
-  substitutable paths is now also shown.  For instance, a build will
-  show something like
-
-    <screen>
-the following derivations will be built:
-  /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv
-  /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv
-  ...
-the following paths will be downloaded/copied (30.02 MiB):
-  /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
-  /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
-  ...</screen>
-
-  </para></listitem>
-
-  <listitem><para>Language features:
-
-    <itemizedlist>
-
-      <listitem><para>@-patterns as in Haskell.  For instance, in a
-      function definition
-
-      <programlisting>f = args @ {x, y, z}: <emphasis>...</emphasis>;</programlisting>
-
-      <varname>args</varname> refers to the argument as a whole, which
-      is further pattern-matched against the attribute set pattern
-      <literal>{x, y, z}</literal>.</para></listitem>
-
-      <listitem><para>“<literal>...</literal>” (ellipsis) patterns.
-      An attribute set pattern can now say <literal>...</literal>  at
-      the end of the attribute name list to specify that the function
-      takes <emphasis>at least</emphasis> the listed attributes, while
-      ignoring additional attributes.  For instance,
-
-      <programlisting>{stdenv, fetchurl, fuse, ...}: <emphasis>...</emphasis></programlisting>
-
-      defines a function that accepts any attribute set that includes
-      at least the three listed attributes.</para></listitem>
-
-      <listitem><para>New primops:
-      <varname>builtins.parseDrvName</varname> (split a package name
-      string like <literal>"nix-0.12pre12876"</literal> into its name
-      and version components, e.g. <literal>"nix"</literal> and
-      <literal>"0.12pre12876"</literal>),
-      <varname>builtins.compareVersions</varname> (compare two version
-      strings using the same algorithm that <command>nix-env</command>
-      uses), <varname>builtins.length</varname> (efficiently compute
-      the length of a list), <varname>builtins.mul</varname> (integer
-      multiplication), <varname>builtins.div</varname> (integer
-      division).
-      <!-- <varname>builtins.genericClosure</varname> -->
-      </para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-  <listitem><para><command>nix-prefetch-url</command> now supports
-  <literal>mirror://</literal> URLs, provided that the environment
-  variable <literal>NIXPKGS_ALL</literal> points at a Nixpkgs
-  tree.</para></listitem>
-
-  <listitem><para>Removed the commands
-  <command>nix-pack-closure</command> and
-  <command>nix-unpack-closure</command>.   You can do almost the same
-  thing but much more efficiently by doing <literal>nix-store --export
-  $(nix-store -qR <emphasis>paths</emphasis>) > closure</literal> and
-  <literal>nix-store --import &lt;
-  closure</literal>.</para></listitem>
-
-  <listitem><para>Lots of bug fixes, including a big performance bug in
-  the handling of <literal>with</literal>-expressions.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.13.xml b/doc/manual/release-notes/rl-0.13.xml
deleted file mode 100644
index 8cb0ae9a5..000000000
--- a/doc/manual/release-notes/rl-0.13.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-0.13">
-
-<title>Release 0.13 (2009-11-05)</title>
-
-<para>This is primarily a bug fix release.  It has some new
-features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Syntactic sugar for writing nested attribute sets.  Instead of
-
-<programlisting>
-{
-  foo = {
-    bar = 123;
-    xyzzy = true;
-  };
-  a = { b = { c = "d"; }; };
-}
-</programlisting>
-
-    you can write
-
-<programlisting>
-{
-  foo.bar = 123;
-  foo.xyzzy = true;
-  a.b.c = "d";
-}
-</programlisting>
-
-    This is useful, for instance, in NixOS configuration files.</para>
-
-  </listitem>
-
-  <listitem>
-    <para>Support for Nix channels generated by Hydra, the Nix-based
-    continuous build system.  (Hydra generates NAR archives on the
-    fly, so the size and hash of these archives isn’t known in
-    advance.)</para>
-  </listitem>
-
-  <listitem>
-    <para>Support <literal>i686-linux</literal> builds directly on
-    <literal>x86_64-linux</literal> Nix installations.  This is
-    implemented using the <function>personality()</function> syscall,
-    which causes <command>uname</command> to return
-    <literal>i686</literal> in child processes.</para>
-  </listitem>
-
-  <listitem>
-    <para>Various improvements to the <literal>chroot</literal>
-    support.  Building in a <literal>chroot</literal> works quite well
-    now.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix no longer blocks if it tries to build a path and another
-    process is already building the same path.  Instead it tries to
-    build another buildable path first.  This improves
-    parallelism.</para>
-  </listitem>
-
-  <listitem>
-    <para>Support for large (> 4 GiB) files in NAR archives.</para>
-  </listitem>
-
-  <listitem>
-    <para>Various (performance) improvements to the remote build
-    mechanism.</para>
-  </listitem>
-
-  <listitem>
-    <para>New primops: <varname>builtins.addErrorContext</varname> (to
-    add a string to stack traces — useful for debugging),
-    <varname>builtins.isBool</varname>,
-    <varname>builtins.isString</varname>,
-    <varname>builtins.isInt</varname>,
-    <varname>builtins.intersectAttrs</varname>.</para>
-  </listitem>
-
-  <listitem>
-    <para>OpenSolaris support (Sander van der Burg).</para>
-  </listitem>
-
-  <listitem>
-    <para>Stack traces are no longer displayed unless the
-    <option>--show-trace</option> option is used.</para>
-  </listitem>
-
-  <listitem>
-    <para>The scoping rules for <literal>inherit
-    (<emphasis>e</emphasis>) ...</literal> in recursive
-    attribute sets have changed.  The expression
-    <emphasis>e</emphasis> can now refer to the attributes
-    defined in the containing set.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.14.xml b/doc/manual/release-notes/rl-0.14.xml
deleted file mode 100644
index e5fe9da78..000000000
--- a/doc/manual/release-notes/rl-0.14.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-0.14">
-
-<title>Release 0.14 (2010-02-04)</title>
-
-<para>This release has the following improvements:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>The garbage collector now starts deleting garbage much
-    faster than before.  It no longer determines liveness of all paths
-    in the store, but does so on demand.</para>
-  </listitem>
-
-  <listitem>
-    <para>Added a new operation, <command>nix-store --query
-    --roots</command>, that shows the garbage collector roots that
-    directly or indirectly point to the given store paths.</para>
-  </listitem>
-
-  <listitem>
-    <para>Removed support for converting Berkeley DB-based Nix
-    databases to the new schema.</para>
-  </listitem>
-
-  <listitem>
-    <para>Removed the <option>--use-atime</option> and
-    <option>--max-atime</option> garbage collector options.  They were
-    not very useful in practice.</para>
-  </listitem>
-
-  <listitem>
-    <para>On Windows, Nix now requires Cygwin 1.7.x.</para>
-  </listitem>
-
-  <listitem>
-    <para>A few bug fixes.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.15.xml b/doc/manual/release-notes/rl-0.15.xml
deleted file mode 100644
index 9f58a8efc..000000000
--- a/doc/manual/release-notes/rl-0.15.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-0.15">
-
-<title>Release 0.15 (2010-03-17)</title>
-
-<para>This is a bug-fix release.  Among other things, it fixes
-building on Mac OS X (Snow Leopard), and improves the contents of
-<filename>/etc/passwd</filename> and <filename>/etc/group</filename>
-in <literal>chroot</literal> builds.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.16.xml b/doc/manual/release-notes/rl-0.16.xml
deleted file mode 100644
index 7039d4b0b..000000000
--- a/doc/manual/release-notes/rl-0.16.xml
+++ /dev/null
@@ -1,55 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-0.16">
-
-<title>Release 0.16 (2010-08-17)</title>
-
-<para>This release has the following improvements:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>The Nix expression evaluator is now much faster in most
-    cases: typically, <link
-    xlink:href="http://www.mail-archive.com/nix-dev@cs.uu.nl/msg04113.html">3
-    to 8 times compared to the old implementation</link>.  It also
-    uses less memory.  It no longer depends on the ATerm
-    library.</para>
-  </listitem>
-
-  <listitem>
-    <para>
-      Support for configurable parallelism inside builders.  Build
-      scripts have always had the ability to perform multiple build
-      actions in parallel (for instance, by running <command>make -j
-      2</command>), but this was not desirable because the number of
-      actions to be performed in parallel was not configurable.  Nix
-      now has an option <option>--cores
-      <emphasis>N</emphasis></option> as well as a configuration
-      setting <varname>build-cores =
-      <emphasis>N</emphasis></varname> that causes the
-      environment variable <literal>NIX_BUILD_CORES</literal> to be set to
-      <emphasis>N</emphasis> when the builder is invoked.  The
-      builder can use this at its discretion to perform a parallel
-      build, e.g., by calling <command>make -j
-      <emphasis>N</emphasis></command>.  In Nixpkgs, this can be
-      enabled on a per-package basis by setting the derivation
-      attribute <varname>enableParallelBuilding</varname> to
-      <literal>true</literal>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-store -q</command> now supports XML output
-    through the <option>--xml</option> flag.</para>
-  </listitem>
-
-  <listitem>
-    <para>Several bug fixes.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.5.xml b/doc/manual/release-notes/rl-0.5.xml
deleted file mode 100644
index e9f8bf270..000000000
--- a/doc/manual/release-notes/rl-0.5.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.5">
-
-<title>Release 0.5 and earlier</title>
-
-<para>Please refer to the Subversion commit log messages.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.6.xml b/doc/manual/release-notes/rl-0.6.xml
deleted file mode 100644
index 23fbee173..000000000
--- a/doc/manual/release-notes/rl-0.6.xml
+++ /dev/null
@@ -1,122 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.6">
-
-<title>Release 0.6 (2004-11-14)</title>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Rewrite of the normalisation engine.
-
-    <itemizedlist>
-
-      <listitem><para>Multiple builds can now be performed in parallel
-      (option <option>-j</option>).</para></listitem>
-
-      <listitem><para>Distributed builds.  Nix can now call a shell
-      script to forward builds to Nix installations on remote
-      machines, which may or may not be of the same platform
-      type.</para></listitem>
-
-      <listitem><para>Option <option>--fallback</option> allows
-      recovery from broken substitutes.</para></listitem>
-
-      <listitem><para>Option <option>--keep-going</option> causes
-      building of other (unaffected) derivations to continue if one
-      failed.</para></listitem>
-
-    </itemizedlist>
-
-    </para>
-
-  </listitem>
-
-  <listitem><para>Improvements to the garbage collector (i.e., it
-  should actually work now).</para></listitem>
-
-  <listitem><para>Setuid Nix installations allow a Nix store to be
-  shared among multiple users.</para></listitem>
-
-  <listitem><para>Substitute registration is much faster
-  now.</para></listitem>
-
-  <listitem><para>A utility <command>nix-build</command> to build a
-  Nix expression and create a symlink to the result int the current
-  directory; useful for testing Nix derivations.</para></listitem>
-
-  <listitem><para>Manual updates.</para></listitem>
-
-  <listitem>
-
-    <para><command>nix-env</command> changes:
-
-    <itemizedlist>
-
-      <listitem><para>Derivations for other platforms are filtered out
-      (which can be overridden using
-      <option>--system-filter</option>).</para></listitem>
-
-      <listitem><para><option>--install</option> by default now
-      uninstall previous derivations with the same
-      name.</para></listitem>
-
-      <listitem><para><option>--upgrade</option> allows upgrading to a
-      specific version.</para></listitem>
-
-      <listitem><para>New operation
-      <option>--delete-generations</option> to remove profile
-      generations (necessary for effective garbage
-      collection).</para></listitem>
-
-      <listitem><para>Nicer output (sorted,
-      columnised).</para></listitem>
-
-    </itemizedlist>
-
-    </para>
-
-  </listitem>
-
-  <listitem><para>More sensible verbosity levels all around (builder
-  output is now shown always, unless <option>-Q</option> is
-  given).</para></listitem>
-
-  <listitem>
-
-    <para>Nix expression language changes:
-
-    <itemizedlist>
-
-      <listitem><para>New language construct: <literal>with
-      <emphasis>E1</emphasis>;
-      <emphasis>E2</emphasis></literal> brings all attributes
-      defined in the attribute set <emphasis>E1</emphasis> in
-      scope in <emphasis>E2</emphasis>.</para></listitem>
-
-      <listitem><para>Added a <function>map</function>
-      function.</para></listitem>
-
-      <listitem><para>Various new operators (e.g., string
-      concatenation).</para></listitem>
-
-    </itemizedlist>
-
-    </para>
-
-  </listitem>
-
-  <listitem><para>Expression evaluation is much
-  faster.</para></listitem>
-
-  <listitem><para>An Emacs mode for editing Nix expressions (with
-  syntax highlighting and indentation) has been
-  added.</para></listitem>
-
-  <listitem><para>Many bug fixes.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.7.xml b/doc/manual/release-notes/rl-0.7.xml
deleted file mode 100644
index 6f95db436..000000000
--- a/doc/manual/release-notes/rl-0.7.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.7">
-
-<title>Release 0.7 (2005-01-12)</title>
-
-<itemizedlist>
-
-  <listitem><para>Binary patching.  When upgrading components using
-  pre-built binaries (through nix-pull / nix-channel), Nix can
-  automatically download and apply binary patches to already installed
-  components instead of full downloads.  Patching is “smart”: if there
-  is a <emphasis>sequence</emphasis> of patches to an installed
-  component, Nix will use it.  Patches are currently generated
-  automatically between Nixpkgs (pre-)releases.</para></listitem>
-
-  <listitem><para>Simplifications to the substitute
-  mechanism.</para></listitem>
-
-  <listitem><para>Nix-pull now stores downloaded manifests in
-  <filename>/nix/var/nix/manifests</filename>.</para></listitem>
-
-  <listitem><para>Metadata on files in the Nix store is canonicalised
-  after builds: the last-modified timestamp is set to 0 (00:00:00
-  1/1/1970), the mode is set to 0444 or 0555 (readable and possibly
-  executable by all; setuid/setgid bits are dropped), and the group is
-  set to the default.  This ensures that the result of a build and an
-  installation through a substitute is the same; and that timestamp
-  dependencies are revealed.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.8.1.xml b/doc/manual/release-notes/rl-0.8.1.xml
deleted file mode 100644
index f7ffca0f8..000000000
--- a/doc/manual/release-notes/rl-0.8.1.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.8.1">
-
-<title>Release 0.8.1 (2005-04-13)</title>
-
-<para>This is a bug fix release.</para>
-
-<itemizedlist>
-
-  <listitem><para>Patch downloading was broken.</para></listitem>
-
-  <listitem><para>The garbage collector would not delete paths that
-  had references from invalid (but substitutable)
-  paths.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml
deleted file mode 100644
index 1adb91a23..000000000
--- a/doc/manual/release-notes/rl-0.8.xml
+++ /dev/null
@@ -1,246 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.8">
-
-<title>Release 0.8 (2005-04-11)</title>
-
-<para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
-As a result, <command>nix-pull</command> manifests and channels built
-for Nix 0.7 and below will not work anymore.  However, the Nix
-expression language has not changed, so you can still build from
-source.  Also, existing user environments continue to work.  Nix 0.8
-will automatically upgrade the database schema of previous
-installations when it is first run.</para>
-
-<para>If you get the error message
-
-<screen>
-you have an old-style manifest `/nix/var/nix/manifests/[...]'; please
-delete it</screen>
-
-you should delete previously downloaded manifests:
-
-<screen>
-$ rm /nix/var/nix/manifests/*</screen>
-
-If <command>nix-channel</command> gives the error message
-
-<screen>
-manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
-is too old (i.e., for Nix &lt;= 0.7)</screen>
-
-then you should unsubscribe from the offending channel
-(<command>nix-channel --remove
-<emphasis>URL</emphasis></command>; leave out
-<literal>/MANIFEST</literal>), and subscribe to the same URL, with
-<literal>channels</literal> replaced by <literal>channels-v3</literal>
-(e.g., <link
-xlink:href='http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable'
-/>).</para>
-
-<para>Nix 0.8 has the following improvements:
-
-<itemizedlist>
-
-  <listitem><para>The cryptographic hashes used in store paths are now
-  160 bits long, but encoded in base-32 so that they are still only 32
-  characters long (e.g.,
-  <filename>/nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0</filename>).
-  (This is actually a 160 bit truncation of a SHA-256
-  hash.)</para></listitem>
-
-  <listitem><para>Big cleanups and simplifications of the basic store
-  semantics.  The notion of “closure store expressions” is gone (and
-  so is the notion of “successors”); the file system references of a
-  store path are now just stored in the database.</para>
-
-  <para>For instance, given any store path, you can query its closure:
-
-  <screen>
-$ nix-store -qR $(which firefox)
-... lots of paths ...</screen>
-
-  Also, Nix now remembers for each store path the derivation that
-  built it (the “deriver”):
-
-  <screen>
-$ nix-store -qR $(which firefox)
-/nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv</screen>
-
-  So to see the build-time dependencies, you can do
-
-  <screen>
-$ nix-store -qR $(nix-store -qd $(which firefox))</screen>
-
-  or, in a nicer format:
-
-  <screen>
-$ nix-store -q --tree $(nix-store -qd $(which firefox))</screen>
-
-  </para>
-
-  <para>File system references are also stored in reverse.  For
-  instance, you can query all paths that directly or indirectly use a
-  certain Glibc:
-
-  <screen>
-$ nix-store -q --referrers-closure \
-    /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4</screen>
-
-  </para>
-
-  </listitem>
-
-  <listitem><para>The concept of fixed-output derivations has been
-  formalised.  Previously, functions such as
-  <function>fetchurl</function> in Nixpkgs used a hack (namely,
-  explicitly specifying a store path hash) to prevent changes to, say,
-  the URL of the file from propagating upwards through the dependency
-  graph, causing rebuilds of everything.  This can now be done cleanly
-  by specifying the <varname>outputHash</varname> and
-  <varname>outputHashAlgo</varname> attributes.  Nix itself checks
-  that the content of the output has the specified hash.  (This is
-  important for maintaining certain invariants necessary for future
-  work on secure shared stores.)</para></listitem>
-
-  <listitem><para>One-click installation :-) It is now possible to
-  install any top-level component in Nixpkgs directly, through the web
-  — see, e.g., <link
-  xlink:href='http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/' />.
-  All you have to do is associate
-  <filename>/nix/bin/nix-install-package</filename> with the MIME type
-  <literal>application/nix-package</literal> (or the extension
-  <filename>.nixpkg</filename>), and clicking on a package link will
-  cause it to be installed, with all appropriate dependencies.  If you
-  just want to install some specific application, this is easier than
-  subscribing to a channel.</para></listitem>
-
-  <listitem><para><command>nix-store -r
-  <emphasis>PATHS</emphasis></command> now builds all the
-  derivations PATHS in parallel.  Previously it did them sequentially
-  (though exploiting possible parallelism between subderivations).
-  This is nice for build farms.</para></listitem>
-
-  <listitem><para><command>nix-channel</command> has new operations
-  <option>--list</option> and
-  <option>--remove</option>.</para></listitem>
-
-  <listitem><para>New ways of installing components into user
-  environments:
-
-  <itemizedlist>
-
-    <listitem><para>Copy from another user environment:
-
-    <screen>
-$ nix-env -i --from-profile .../other-profile firefox</screen>
-
-    </para></listitem>
-
-    <listitem><para>Install a store derivation directly (bypassing the
-    Nix expression language entirely):
-
-    <screen>
-$ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv</screen>
-
-    (This is used to implement <command>nix-install-package</command>,
-    which is therefore immune to evolution in the Nix expression
-    language.)</para></listitem>
-
-    <listitem><para>Install an already built store path directly:
-
-    <screen>
-$ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1</screen>
-
-    </para></listitem>
-
-    <listitem><para>Install the result of a Nix expression specified
-    as a command-line argument:
-
-    <screen>
-$ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'</screen>
-
-    The difference with the normal installation mode is that
-    <option>-E</option> does not use the <varname>name</varname>
-    attributes of derivations.  Therefore, this can be used to
-    disambiguate multiple derivations with the same
-    name.</para></listitem>
-
-  </itemizedlist></para></listitem>
-
-  <listitem><para>A hash of the contents of a store path is now stored
-  in the database after a successful build.  This allows you to check
-  whether store paths have been tampered with: <command>nix-store
-  --verify --check-contents</command>.</para></listitem>
-
-  <listitem>
-
-    <para>Implemented a concurrent garbage collector.  It is now
-    always safe to run the garbage collector, even if other Nix
-    operations are happening simultaneously.</para>
-
-    <para>However, there can still be GC races if you use
-    <command>nix-instantiate</command> and <command>nix-store
-    --realise</command> directly to build things.  To prevent races,
-    use the <option>--add-root</option> flag of those commands.</para>
-
-  </listitem>
-
-  <listitem><para>The garbage collector now finally deletes paths in
-  the right order (i.e., topologically sorted under the “references”
-  relation), thus making it safe to interrupt the collector without
-  risking a store that violates the closure
-  invariant.</para></listitem>
-
-  <listitem><para>Likewise, the substitute mechanism now downloads
-  files in the right order, thus preserving the closure invariant at
-  all times.</para></listitem>
-
-  <listitem><para>The result of <command>nix-build</command> is now
-  registered as a root of the garbage collector.  If the
-  <filename>./result</filename> link is deleted, the GC root
-  disappears automatically.</para></listitem>
-
-  <listitem>
-
-    <para>The behaviour of the garbage collector can be changed
-    globally by setting options in
-    <filename>/nix/etc/nix/nix.conf</filename>.
-
-    <itemizedlist>
-
-      <listitem><para><literal>gc-keep-derivations</literal> specifies
-      whether deriver links should be followed when searching for live
-      paths.</para></listitem>
-
-      <listitem><para><literal>gc-keep-outputs</literal> specifies
-      whether outputs of derivations should be followed when searching
-      for live paths.</para></listitem>
-
-      <listitem><para><literal>env-keep-derivations</literal>
-      specifies whether user environments should store the paths of
-      derivations when they are added (thus keeping the derivations
-      alive).</para></listitem>
-
-    </itemizedlist>
-
-  </para></listitem>
-
-  <listitem><para>New <command>nix-env</command> query flags
-  <option>--drv-path</option> and
-  <option>--out-path</option>.</para></listitem>
-
-  <listitem><para><command>fetchurl</command> allows SHA-1 and SHA-256
-  in addition to MD5.  Just specify the attribute
-  <varname>sha1</varname> or <varname>sha256</varname> instead of
-  <varname>md5</varname>.</para></listitem>
-
-  <listitem><para>Manual updates.</para></listitem>
-
-</itemizedlist>
-
-</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.9.1.xml b/doc/manual/release-notes/rl-0.9.1.xml
deleted file mode 100644
index 85d11f416..000000000
--- a/doc/manual/release-notes/rl-0.9.1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.9.1">
-
-<title>Release 0.9.1 (2005-09-20)</title>
-
-<para>This bug fix release addresses a problem with the ATerm library
-when the <option>--with-aterm</option> flag in
-<command>configure</command> was <emphasis>not</emphasis> used.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.9.2.xml b/doc/manual/release-notes/rl-0.9.2.xml
deleted file mode 100644
index cb705e98a..000000000
--- a/doc/manual/release-notes/rl-0.9.2.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.9.2">
-
-<title>Release 0.9.2 (2005-09-21)</title>
-
-<para>This bug fix release fixes two problems on Mac OS X:
-
-<itemizedlist>
-
-  <listitem><para>If Nix was linked against statically linked versions
-  of the ATerm or Berkeley DB library, there would be dynamic link
-  errors at runtime.</para></listitem>
-
-  <listitem><para><command>nix-pull</command> and
-  <command>nix-push</command> intermittently failed due to race
-  conditions involving pipes and child processes with error messages
-  such as <literal>open2: open(GLOB(0x180b2e4), >&amp;=9) failed: Bad
-  file descriptor at /nix/bin/nix-pull line 77</literal> (issue
-  <literal>NIX-14</literal>).</para></listitem>
-
-</itemizedlist>
-
-</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-0.9.xml b/doc/manual/release-notes/rl-0.9.xml
deleted file mode 100644
index fd1e633f7..000000000
--- a/doc/manual/release-notes/rl-0.9.xml
+++ /dev/null
@@ -1,98 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ch-relnotes-0.9">
-
-<title>Release 0.9 (2005-09-16)</title>
-
-<para>NOTE: this version of Nix uses Berkeley DB 4.3 instead of 4.2.
-The database is upgraded automatically, but you should be careful not
-to use old versions of Nix that still use Berkeley DB 4.2.  In
-particular, if you use a Nix installed through Nix, you should run
-
-<screen>
-$ nix-store --clear-substitutes</screen>
-
-first.</para>
-
-
-<itemizedlist>
-
-  <listitem><para>Unpacking of patch sequences is much faster now
-  since we no longer do redundant unpacking and repacking of
-  intermediate paths.</para></listitem>
-
-  <listitem><para>Nix now uses Berkeley DB 4.3.</para></listitem>
-
-  <listitem><para>The <function>derivation</function> primitive is
-  lazier.  Attributes of dependent derivations can mutually refer to
-  each other (as long as there are no data dependencies on the
-  <varname>outPath</varname> and <varname>drvPath</varname> attributes
-  computed by <function>derivation</function>).</para>
-
-  <para>For example, the expression <literal>derivation
-  attrs</literal> now evaluates to (essentially)
-
-  <programlisting>
-attrs // {
-  type = "derivation";
-  outPath = derivation! attrs;
-  drvPath = derivation! attrs;
-}</programlisting>
-
-  where <function>derivation!</function> is a primop that does the
-  actual derivation instantiation (i.e., it does what
-  <function>derivation</function> used to do).  The advantage is that
-  it allows commands such as <command>nix-env -qa</command> and
-  <command>nix-env -i</command> to be much faster since they no longer
-  need to instantiate all derivations, just the
-  <varname>name</varname> attribute.</para>
-
-  <para>Also, it allows derivations to cyclically reference each
-  other, for example,
-
-  <programlisting>
-webServer = derivation {
-  ...
-  hostName = "svn.cs.uu.nl";
-  services = [svnService];
-};
-&#x20;
-svnService = derivation {
-  ...
-  hostName = webServer.hostName;
-};</programlisting>
-
-  Previously, this would yield a black hole (infinite recursion).</para>
-
-  </listitem>
-
-  <listitem><para><command>nix-build</command> now defaults to using
-  <filename>./default.nix</filename> if no Nix expression is
-  specified.</para></listitem>
-
-  <listitem><para><command>nix-instantiate</command>, when applied to
-  a Nix expression that evaluates to a function, will call the
-  function automatically if all its arguments have
-  defaults.</para></listitem>
-
-  <listitem><para>Nix now uses libtool to build dynamic libraries.
-  This reduces the size of executables.</para></listitem>
-
-  <listitem><para>A new list concatenation operator
-  <literal>++</literal>.  For example, <literal>[1 2 3] ++ [4 5
-  6]</literal> evaluates to <literal>[1 2 3 4 5
-  6]</literal>.</para></listitem>
-
-  <listitem><para>Some currently undocumented primops to support
-  low-level build management using Nix (i.e., using Nix as a Make
-  replacement).  See the commit messages for <literal>r3578</literal>
-  and <literal>r3580</literal>.</para></listitem>
-
-  <listitem><para>Various bug fixes and performance
-  improvements.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.0.xml b/doc/manual/release-notes/rl-1.0.xml
deleted file mode 100644
index 41081a19f..000000000
--- a/doc/manual/release-notes/rl-1.0.xml
+++ /dev/null
@@ -1,119 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.0">
-
-<title>Release 1.0 (2012-05-11)</title>
-
-<para>There have been numerous improvements and bug fixes since the
-previous release.  Here are the most significant:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Nix can now optionally use the Boehm garbage collector.
-    This significantly reduces the Nix evaluator’s memory footprint,
-    especially when evaluating large NixOS system configurations.  It
-    can be enabled using the <option>--enable-gc</option> configure
-    option.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now uses SQLite for its database.  This is faster and
-    more flexible than the old <emphasis>ad hoc</emphasis> format.
-    SQLite is also used to cache the manifests in
-    <filename>/nix/var/nix/manifests</filename>, resulting in a
-    significant speedup.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now has an search path for expressions.  The search path
-    is set using the environment variable <literal>NIX_PATH</literal> and
-    the <option>-I</option> command line option.  In Nix expressions,
-    paths between angle brackets are used to specify files that must
-    be looked up in the search path.  For instance, the expression
-    <literal>&lt;nixpkgs/default.nix></literal> looks for a file
-    <filename>nixpkgs/default.nix</filename> relative to every element
-    in the search path.</para>
-  </listitem>
-
-  <listitem>
-    <para>The new command <command>nix-build --run-env</command>
-    builds all dependencies of a derivation, then starts a shell in an
-    environment containing all variables from the derivation.  This is
-    useful for reproducing the environment of a derivation for
-    development.</para>
-  </listitem>
-
-  <listitem>
-    <para>The new command <command>nix-store --verify-path</command>
-    verifies that the contents of a store path have not
-    changed.</para>
-  </listitem>
-
-  <listitem>
-    <para>The new command <command>nix-store --print-env</command>
-    prints out the environment of a derivation in a format that can be
-    evaluated by a shell.</para>
-  </listitem>
-
-  <listitem>
-    <para>Attribute names can now be arbitrary strings.  For instance,
-    you can write <literal>{ "foo-1.2" = …; "bla bla" = …; }."bla
-    bla"</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Attribute selection can now provide a default value using
-    the <literal>or</literal> operator.  For instance, the expression
-    <literal>x.y.z or e</literal> evaluates to the attribute
-    <literal>x.y.z</literal> if it exists, and <literal>e</literal>
-    otherwise.</para>
-  </listitem>
-
-  <listitem>
-    <para>The right-hand side of the <literal>?</literal> operator can
-    now be an attribute path, e.g., <literal>attrs ?
-    a.b.c</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para>On Linux, Nix will now make files in the Nix store immutable
-    on filesystems that support it.  This prevents accidental
-    modification of files in the store by the root user.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix has preliminary support for derivations with multiple
-    outputs.  This is useful because it allows parts of a package to
-    be deployed and garbage-collected separately.  For instance,
-    development parts of a package such as header files or static
-    libraries would typically not be part of the closure of an
-    application, resulting in reduced disk usage and installation
-    time.</para>
-  </listitem>
-
-  <listitem>
-    <para>The Nix store garbage collector is faster and holds the
-    global lock for a shorter amount of time.</para>
-  </listitem>
-
-  <listitem>
-    <para>The option <option>--timeout</option> (corresponding to the
-    configuration setting <literal>build-timeout</literal>) allows you
-    to set an absolute timeout on builds — if a build runs for more than
-    the given number of seconds, it is terminated.  This is useful for
-    recovering automatically from builds that are stuck in an infinite
-    loop but keep producing output, and for which
-    <literal>--max-silent-time</literal> is ineffective.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix development has moved to GitHub (<link
-    xlink:href="https://github.com/NixOS/nix" />).</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.1.xml b/doc/manual/release-notes/rl-1.1.xml
deleted file mode 100644
index f7dadb7cb..000000000
--- a/doc/manual/release-notes/rl-1.1.xml
+++ /dev/null
@@ -1,100 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.1">
-
-<title>Release 1.1 (2012-07-18)</title>
-
-<para>This release has the following improvements:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>On Linux, when doing a chroot build, Nix now uses various
-    namespace features provided by the Linux kernel to improve
-    build isolation.  Namely:
-    <itemizedlist>
-      <listitem><para>The private network namespace ensures that
-      builders cannot talk to the outside world (or vice versa): each
-      build only sees a private loopback interface.  This also means
-      that two concurrent builds can listen on the same port (e.g. as
-      part of a test) without conflicting with each
-      other.</para></listitem>
-      <listitem><para>The PID namespace causes each build to start as
-      PID 1.  Processes outside of the chroot are not visible to those
-      on the inside.  On the other hand, processes inside the chroot
-      <emphasis>are</emphasis> visible from the outside (though with
-      different PIDs).</para></listitem>
-      <listitem><para>The IPC namespace prevents the builder from
-      communicating with outside processes using SysV IPC mechanisms
-      (shared memory, message queues, semaphores).  It also ensures
-      that all IPC objects are destroyed when the builder
-      exits.</para></listitem>
-      <listitem><para>The UTS namespace ensures that builders see a
-      hostname of <literal>localhost</literal> rather than the actual
-      hostname.</para></listitem>
-      <listitem><para>The private mount namespace was already used by
-      Nix to ensure that the bind-mounts used to set up the chroot are
-      cleaned up automatically.</para></listitem>
-    </itemizedlist>
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>Build logs are now compressed using
-    <command>bzip2</command>.  The command <command>nix-store
-    -l</command> decompresses them on the fly.  This can be disabled
-    by setting the option <literal>build-compress-log</literal> to
-    <literal>false</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para>The creation of build logs in
-    <filename>/nix/var/log/nix/drvs</filename> can be disabled by
-    setting the new option <literal>build-keep-log</literal> to
-    <literal>false</literal>.  This is useful, for instance, for Hydra
-    build machines.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now reserves some space in
-    <filename>/nix/var/nix/db/reserved</filename> to ensure that the
-    garbage collector can run successfully if the disk is full.  This
-    is necessary because SQLite transactions fail if the disk is
-    full.</para>
-  </listitem>
-
-  <listitem>
-    <para>Added a basic <function>fetchurl</function> function.  This
-    is not intended to replace the <function>fetchurl</function> in
-    Nixpkgs, but is useful for bootstrapping; e.g., it will allow us
-    to get rid of the bootstrap binaries in the Nixpkgs source tree
-    and download them instead.  You can use it by doing
-    <literal>import &lt;nix/fetchurl.nix> { url =
-    <emphasis>url</emphasis>; sha256 =
-    "<emphasis>hash</emphasis>"; }</literal>. (Shea Levy)</para>
-  </listitem>
-
-  <listitem>
-    <para>Improved RPM spec file. (Michel Alexandre Salim)</para>
-  </listitem>
-
-  <listitem>
-    <para>Support for on-demand socket-based activation in the Nix
-    daemon with <command>systemd</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Added a manpage for
-    <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
-  </listitem>
-
-  <listitem>
-    <para>When using the Nix daemon, the <option>-s</option> flag in
-    <command>nix-env -qa</command> is now much faster.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.10.xml b/doc/manual/release-notes/rl-1.10.xml
deleted file mode 100644
index 689a95466..000000000
--- a/doc/manual/release-notes/rl-1.10.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.10">
-
-<title>Release 1.10 (2015-09-03)</title>
-
-<para>This is primarily a bug fix release. It also has a number of new
-features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>A number of builtin functions have been added to reduce
-    Nixpkgs/NixOS evaluation time and memory consumption:
-    <function>all</function>,
-    <function>any</function>,
-    <function>concatStringsSep</function>,
-    <function>foldl’</function>,
-    <function>genList</function>,
-    <function>replaceStrings</function>,
-    <function>sort</function>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The garbage collector is more robust when the disk is full.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix supports a new API for building derivations that doesn’t
-    require a <literal>.drv</literal> file to be present on disk; it
-    only requires an in-memory representation of the derivation. This
-    is used by the Hydra continuous build system to make remote builds
-    more efficient.</para>
-  </listitem>
-
-  <listitem>
-    <para>The function <literal>&lt;nix/fetchurl.nix></literal> now
-    uses a <emphasis>builtin</emphasis> builder (i.e. it doesn’t
-    require starting an external process; the download is performed by
-    Nix itself). This ensures that derivation paths don’t change when
-    Nix is upgraded, and obviates the need for ugly hacks to support
-    chroot execution.</para>
-  </listitem>
-
-  <listitem>
-    <para><option>--version -v</option> now prints some configuration
-    information, in particular what compile-time optional features are
-    enabled, and the paths of various directories.</para>
-  </listitem>
-
-  <listitem>
-    <para>Build users have their supplementary groups set correctly.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from Eelco Dolstra, Guillaume
-Maudoux, Iwan Aucamp, Jaka Hudoklin, Kirill Elagin, Ludovic Courtès,
-Manolis Ragkousis, Nicolas B. Pierron and Shea Levy.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.11.10.xml b/doc/manual/release-notes/rl-1.11.10.xml
deleted file mode 100644
index 415388b3e..000000000
--- a/doc/manual/release-notes/rl-1.11.10.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.11.10">
-
-<title>Release 1.11.10 (2017-06-12)</title>
-
-<para>This release fixes a security bug in Nix’s “build user” build
-isolation mechanism. Previously, Nix builders had the ability to
-create setuid binaries owned by a <literal>nixbld</literal>
-user. Such a binary could then be used by an attacker to assume a
-<literal>nixbld</literal> identity and interfere with subsequent
-builds running under the same UID.</para>
-
-<para>To prevent this issue, Nix now disallows builders to create
-setuid and setgid binaries. On Linux, this is done using a seccomp BPF
-filter. Note that this imposes a small performance penalty (e.g. 1%
-when building GNU Hello). Using seccomp, we now also prevent the
-creation of extended attributes and POSIX ACLs since these cannot be
-represented in the NAR format and (in the case of POSIX ACLs) allow
-bypassing regular Nix store permissions. On macOS, the restriction is
-implemented using the existing sandbox mechanism, which now uses a
-minimal “allow all except the creation of setuid/setgid binaries”
-profile when regular sandboxing is disabled. On other platforms, the
-“build user” mechanism is now disabled.</para>
-
-<para>Thanks go to Linus Heckemann for discovering and reporting this
-bug.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.11.xml b/doc/manual/release-notes/rl-1.11.xml
deleted file mode 100644
index 28f6d4ac8..000000000
--- a/doc/manual/release-notes/rl-1.11.xml
+++ /dev/null
@@ -1,141 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.11">
-
-<title>Release 1.11 (2016-01-19)</title>
-
-<para>This is primarily a bug fix release. It also has a number of new
-features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para><command>nix-prefetch-url</command> can now download URLs
-    specified in a Nix expression. For example,
-
-<screen>
-$ nix-prefetch-url -A hello.src
-</screen>
-
-    will prefetch the file specified by the
-    <function>fetchurl</function> call in the attribute
-    <literal>hello.src</literal> from the Nix expression in the
-    current directory, and print the cryptographic hash of the
-    resulting file on stdout. This differs from <literal>nix-build -A
-    hello.src</literal> in that it doesn't verify the hash, and is
-    thus useful when you’re updating a Nix expression.</para>
-
-    <para>You can also prefetch the result of functions that unpack a
-    tarball, such as <function>fetchFromGitHub</function>. For example:
-
-<screen>
-$ nix-prefetch-url --unpack https://github.com/NixOS/patchelf/archive/0.8.tar.gz
-</screen>
-
-    or from a Nix expression:
-
-<screen>
-$ nix-prefetch-url -A nix-repl.src
-</screen>
-
-    </para>
-
-  </listitem>
-
-  <listitem>
-    <para>The builtin function
-    <function>&lt;nix/fetchurl.nix></function> now supports
-    downloading and unpacking NARs. This removes the need to have
-    multiple downloads in the Nixpkgs stdenv bootstrap process (like a
-    separate busybox binary for Linux, or curl/mkdir/sh/bzip2 for
-    Darwin). Now all those files can be combined into a single NAR,
-    optionally compressed using <command>xz</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now supports SHA-512 hashes for verifying fixed-output
-    derivations, and in <function>builtins.hashString</function>.</para>
-  </listitem>
-
-  <listitem>
-    <para>
-      The new flag <option>--option build-repeat
-      <emphasis>N</emphasis></option> will cause every build to
-      be executed <emphasis>N</emphasis>+1 times. If the build
-      output differs between any round, the build is rejected, and the
-      output paths are not registered as valid. This is primarily
-      useful to verify build determinism. (We already had a
-      <option>--check</option> option to repeat a previously succeeded
-      build. However, with <option>--check</option>, non-deterministic
-      builds are registered in the DB. Preventing that is useful for
-      Hydra to ensure that non-deterministic builds don't end up
-      getting published to the binary cache.)
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>
-      The options <option>--check</option> and <option>--option
-      build-repeat <emphasis>N</emphasis></option>, if they
-      detect a difference between two runs of the same derivation and
-      <option>-K</option> is given, will make the output of the other
-      run available under
-      <filename><emphasis>store-path</emphasis>-check</filename>. This
-      makes it easier to investigate the non-determinism using tools
-      like <command>diffoscope</command>, e.g.,
-
-<screen>
-$ nix-build pkgs/stdenv/linux -A stage1.pkgs.zlib --check -K
-error: derivation ‘/nix/store/l54i8wlw2265…-zlib-1.2.8.drv’ may not
-be deterministic: output ‘/nix/store/11a27shh6n2i…-zlib-1.2.8’
-differs from ‘/nix/store/11a27shh6n2i…-zlib-1.2.8-check’
-
-$ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zlib-1.2.8-check
-…
-├── lib/libz.a
-│   ├── metadata
-│   │ @@ -1,15 +1,15 @@
-│   │ -rw-r--r-- 30001/30000   3096 Jan 12 15:20 2016 adler32.o
-…
-│   │ +rw-r--r-- 30001/30000   3096 Jan 12 15:28 2016 adler32.o
-…
-</screen>
-
-    </para></listitem>
-
-  <listitem>
-    <para>Improved FreeBSD support.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env -qa --xml --meta</command> now prints
-    license information.</para>
-  </listitem>
-
-  <listitem>
-    <para>The maximum number of parallel TCP connections that the
-    binary cache substituter will use has been decreased from 150 to
-    25. This should prevent upsetting some broken NAT routers, and
-    also improves performance.</para>
-  </listitem>
-
-  <listitem>
-    <para>All "chroot"-containing strings got renamed to "sandbox".
-      In particular, some Nix options got renamed, but the old names
-      are still accepted as lower-priority aliases.
-    </para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from Anders Claesson, Anthony
-Cowley, Bjørn Forsman, Brian McKenna, Danny Wilson, davidak, Eelco Dolstra,
-Fabian Schmitthenner, FrankHB, Ilya Novoselov, janus, Jim Garrison, John
-Ericson, Jude Taylor, Ludovic Courtès, Manuel Jacob, Mathnerd314,
-Pascal Wittmann, Peter Simons, Philip Potter, Preston Bennes, Rommel
-M. Martinez, Sander van der Burg, Shea Levy, Tim Cuthbertson, Tuomas
-Tynkkynen, Utku Demir and Vladimír Čunát.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.2.xml b/doc/manual/release-notes/rl-1.2.xml
deleted file mode 100644
index f065f3ccc..000000000
--- a/doc/manual/release-notes/rl-1.2.xml
+++ /dev/null
@@ -1,157 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.2">
-
-<title>Release 1.2 (2012-12-06)</title>
-
-<para>This release has the following improvements and changes:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Nix has a new binary substituter mechanism: the
-    <emphasis>binary cache</emphasis>.  A binary cache contains
-    pre-built binaries of Nix packages.  Whenever Nix wants to build a
-    missing Nix store path, it will check a set of binary caches to
-    see if any of them has a pre-built binary of that path.  The
-    configuration setting <option>binary-caches</option> contains a
-    list of URLs of binary caches.  For instance, doing
-<screen>
-$ nix-env -i thunderbird --option binary-caches http://cache.nixos.org
-</screen>
-    will install Thunderbird and its dependencies, using the available
-    pre-built binaries in <uri>http://cache.nixos.org</uri>.
-    The main advantage over the old “manifest”-based method of getting
-    pre-built binaries is that you don’t have to worry about your
-    manifest being in sync with the Nix expressions you’re installing
-    from; i.e., you don’t need to run <command>nix-pull</command> to
-    update your manifest.  It’s also more scalable because you don’t
-    need to redownload a giant manifest file every time.
-    </para>
-
-    <para>A Nix channel can provide a binary cache URL that will be
-    used automatically if you subscribe to that channel.  If you use
-    the Nixpkgs or NixOS channels
-    (<uri>http://nixos.org/channels</uri>) you automatically get the
-    cache <uri>http://cache.nixos.org</uri>.</para>
-
-    <para>Binary caches are created using <command>nix-push</command>.
-    For details on the operation and format of binary caches, see the
-    <command>nix-push</command> manpage.  More details are provided in
-    <link xlink:href="https://nixos.org/nix-dev/2012-September/009826.html">this
-    nix-dev posting</link>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Multiple output support should now be usable.  A derivation
-    can declare that it wants to produce multiple store paths by
-    saying something like
-<programlisting>
-outputs = [ "lib" "headers" "doc" ];
-</programlisting>
-    This will cause Nix to pass the intended store path of each output
-    to the builder through the environment variables
-    <literal>lib</literal>, <literal>headers</literal> and
-    <literal>doc</literal>.  Other packages can refer to a specific
-    output by referring to
-    <literal><emphasis>pkg</emphasis>.<emphasis>output</emphasis></literal>,
-    e.g.
-<programlisting>
-buildInputs = [ pkg.lib pkg.headers ];
-</programlisting>
-    If you install a package with multiple outputs using
-    <command>nix-env</command>, each output path will be symlinked
-    into the user environment.</para>
-  </listitem>
-
-  <listitem>
-    <para>Dashes are now valid as part of identifiers and attribute
-    names.</para>
-  </listitem>
-
-  <listitem>
-    <para>The new operation <command>nix-store --repair-path</command>
-    allows corrupted or missing store paths to be repaired by
-    redownloading them.  <command>nix-store --verify --check-contents
-    --repair</command> will scan and repair all paths in the Nix
-    store.  Similarly, <command>nix-env</command>,
-    <command>nix-build</command>, <command>nix-instantiate</command>
-    and <command>nix-store --realise</command> have a
-    <option>--repair</option> flag to detect and fix bad paths by
-    rebuilding or redownloading them.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix no longer sets the immutable bit on files in the Nix
-    store.  Instead, the recommended way to guard the Nix store
-    against accidental modification on Linux is to make it a read-only
-    bind mount, like this:
-
-<screen>
-$ mount --bind /nix/store /nix/store
-$ mount -o remount,ro,bind /nix/store
-</screen>
-
-    Nix will automatically make <filename>/nix/store</filename>
-    writable as needed (using a private mount namespace) to allow
-    modifications.</para>
-  </listitem>
-
-  <listitem>
-    <para>Store optimisation (replacing identical files in the store
-    with hard links) can now be done automatically every time a path
-    is added to the store.  This is enabled by setting the
-    configuration option <literal>auto-optimise-store</literal> to
-    <literal>true</literal> (disabled by default).</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now supports <command>xz</command> compression for NARs
-    in addition to <command>bzip2</command>.  It compresses about 30%
-    better on typical archives and decompresses about twice as
-    fast.</para>
-  </listitem>
-
-  <listitem>
-    <para>Basic Nix expression evaluation profiling: setting the
-    environment variable <literal>NIX_COUNT_CALLS</literal> to
-    <literal>1</literal> will cause Nix to print how many times each
-    primop or function was executed.</para>
-  </listitem>
-
-  <listitem>
-    <para>New primops: <varname>concatLists</varname>,
-    <varname>elem</varname>, <varname>elemAt</varname> and
-    <varname>filter</varname>.</para>
-  </listitem>
-
-  <listitem>
-    <para>The command <command>nix-copy-closure</command> has a new
-    flag <option>--use-substitutes</option> (<option>-s</option>) to
-    download missing paths on the target machine using the substitute
-    mechanism.</para>
-  </listitem>
-
-  <listitem>
-    <para>The command <command>nix-worker</command> has been renamed
-    to <command>nix-daemon</command>.  Support for running the Nix
-    worker in “slave” mode has been removed.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>--help</option> flag of every Nix command now
-    invokes <command>man</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Chroot builds are now supported on systemd machines.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from Eelco Dolstra, Florian
-Friesdorf, Mats Erik Andersson and Shea Levy.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.3.xml b/doc/manual/release-notes/rl-1.3.xml
deleted file mode 100644
index e2009ee3b..000000000
--- a/doc/manual/release-notes/rl-1.3.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.3">
-
-<title>Release 1.3 (2013-01-04)</title>
-
-<para>This is primarily a bug fix release.  When this version is first
-run on Linux, it removes any immutable bits from the Nix store and
-increases the schema version of the Nix store.  (The previous release
-removed support for setting the immutable bit; this release clears any
-remaining immutable bits to make certain operations more
-efficient.)</para>
-
-<para>This release has contributions from Eelco Dolstra and Stuart
-Pernsteiner.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.4.xml b/doc/manual/release-notes/rl-1.4.xml
deleted file mode 100644
index 8114ce6b5..000000000
--- a/doc/manual/release-notes/rl-1.4.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.4">
-
-<title>Release 1.4 (2013-02-26)</title>
-
-<para>This release fixes a security bug in multi-user operation.  It
-was possible for derivations to cause the mode of files outside of the
-Nix store to be changed to 444 (read-only but world-readable) by
-creating hard links to those files (<link
-xlink:href="https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a79141ac4">details</link>).</para>
-
-<para>There are also the following improvements:</para>
-
-<itemizedlist>
-
-  <listitem><para>New built-in function:
-  <function>builtins.hashString</function>.</para></listitem>
-
-  <listitem><para>Build logs are now stored in
-  <filename>/nix/var/log/nix/drvs/<emphasis>XX</emphasis>/</filename>,
-  where <emphasis>XX</emphasis> is the first two characters of
-  the derivation.  This is useful on machines that keep a lot of build
-  logs (such as Hydra servers).</para></listitem>
-
-  <listitem><para>The function <function>corepkgs/fetchurl</function>
-  can now make the downloaded file executable.  This will allow
-  getting rid of all bootstrap binaries in the Nixpkgs source
-  tree.</para></listitem>
-
-  <listitem><para>Language change: The expression <literal>"${./path}
-  ..."</literal> now evaluates to a string instead of a
-  path.</para></listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.5.1.xml b/doc/manual/release-notes/rl-1.5.1.xml
deleted file mode 100644
index 035c8dbcb..000000000
--- a/doc/manual/release-notes/rl-1.5.1.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.5.1">
-
-<title>Release 1.5.1 (2013-02-28)</title>
-
-<para>The bug fix to the bug fix had a bug itself, of course.  But
-this time it will work for sure!</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.5.2.xml b/doc/manual/release-notes/rl-1.5.2.xml
deleted file mode 100644
index 7e81dd243..000000000
--- a/doc/manual/release-notes/rl-1.5.2.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.5.2">
-
-<title>Release 1.5.2 (2013-05-13)</title>
-
-<para>This is primarily a bug fix release.  It has contributions from
-Eelco Dolstra, Lluís Batlle i Rossell and Shea Levy.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.5.xml b/doc/manual/release-notes/rl-1.5.xml
deleted file mode 100644
index 8e279d769..000000000
--- a/doc/manual/release-notes/rl-1.5.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.5">
-
-<title>Release 1.5 (2013-02-27)</title>
-
-<para>This is a brown paper bag release to fix a regression introduced
-by the hard link security fix in 1.4.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.6.1.xml b/doc/manual/release-notes/rl-1.6.1.xml
deleted file mode 100644
index 982871b51..000000000
--- a/doc/manual/release-notes/rl-1.6.1.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.6.1">
-
-<title>Release 1.6.1 (2013-10-28)</title>
-
-<para>This is primarily a bug fix release.  Changes of interest
-are:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Nix 1.6 accidentally changed the semantics of antiquoted
-    paths in strings, such as <literal>"${/foo}/bar"</literal>.  This
-    release reverts to the Nix 1.5.3 behaviour.</para>
-  </listitem>
-
-  <listitem>
-    <para>Previously, Nix optimised expressions such as
-    <literal>"${<emphasis>expr</emphasis>}"</literal> to
-    <emphasis>expr</emphasis>.  Thus it neither checked whether
-    <emphasis>expr</emphasis> could be coerced to a string, nor
-    applied such coercions.  This meant that
-    <literal>"${123}"</literal> evaluatued to <literal>123</literal>,
-    and <literal>"${./foo}"</literal> evaluated to
-    <literal>./foo</literal> (even though
-    <literal>"${./foo} "</literal> evaluates to
-    <literal>"/nix/store/<emphasis>hash</emphasis>-foo "</literal>).
-    Nix now checks the type of antiquoted expressions and
-    applies coercions.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now shows the exact position of undefined variables.  In
-    particular, undefined variable errors in a <literal>with</literal>
-    previously didn't show <emphasis>any</emphasis> position
-    information, so this makes it a lot easier to fix such
-    errors.</para>
-  </listitem>
-
-  <listitem>
-    <para>Undefined variables are now treated consistently.
-    Previously, the <function>tryEval</function> function would catch
-    undefined variables inside a <literal>with</literal> but not
-    outside.  Now <function>tryEval</function> never catches undefined
-    variables.</para>
-  </listitem>
-
-  <listitem>
-    <para>Bash completion in <command>nix-shell</command> now works
-    correctly.</para>
-  </listitem>
-
-  <listitem>
-    <para>Stack traces are less verbose: they no longer show calls to
-    builtin functions and only show a single line for each derivation
-    on the call stack.</para>
-  </listitem>
-
-  <listitem>
-    <para>New built-in function: <function>builtins.typeOf</function>,
-    which returns the type of its argument as a string.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.6.xml b/doc/manual/release-notes/rl-1.6.xml
deleted file mode 100644
index cdb52ed0b..000000000
--- a/doc/manual/release-notes/rl-1.6.xml
+++ /dev/null
@@ -1,127 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.6.0">
-
-<title>Release 1.6 (2013-09-10)</title>
-
-<para>In addition to the usual bug fixes, this release has several new
-features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>The command <command>nix-build --run-env</command> has been
-    renamed to <command>nix-shell</command>.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> now sources
-    <filename>$stdenv/setup</filename> <emphasis>inside</emphasis> the
-    interactive shell, rather than in a parent shell.  This ensures
-    that shell functions defined by <literal>stdenv</literal> can be
-    used in the interactive shell.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> has a new flag
-    <option>--pure</option> to clear the environment, so you get an
-    environment that more closely corresponds to the “real” Nix build.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> now sets the shell prompt
-    (<literal>PS1</literal>) to ensure that Nix shells are distinguishable
-    from your regular shells.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env</command> no longer requires a
-    <literal>*</literal> argument to match all packages, so
-    <literal>nix-env -qa</literal> is equivalent to <literal>nix-env
-    -qa '*'</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env -i</command> has a new flag
-    <option>--remove-all</option> (<option>-r</option>) to remove all
-    previous packages from the profile.  This makes it easier to do
-    declarative package management similar to NixOS’s
-    <option>environment.systemPackages</option>.  For instance, if you
-    have a specification <filename>my-packages.nix</filename> like this:
-
-<programlisting>
-with import &lt;nixpkgs> {};
-[ thunderbird
-  geeqie
-  ...
-]
-</programlisting>
-
-    then after any change to this file, you can run:
-
-<screen>
-$ nix-env -f my-packages.nix -ir
-</screen>
-
-    to update your profile to match the specification.</para>
-  </listitem>
-
-  <listitem>
-    <para>The ‘<literal>with</literal>’ language construct is now more
-    lazy.  It only evaluates its argument if a variable might actually
-    refer to an attribute in the argument.  For instance, this now
-    works:
-
-<programlisting>
-let
-  pkgs = with pkgs; { foo = "old"; bar = foo; } // overrides;
-  overrides = { foo = "new"; };
-in pkgs.bar
-</programlisting>
-
-    This evaluates to <literal>"new"</literal>, while previously it
-    gave an “infinite recursion” error.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now has proper integer arithmetic operators. For
-    instance, you can write <literal>x + y</literal> instead of
-    <literal>builtins.add x y</literal>, or <literal>x &lt;
-    y</literal> instead of <literal>builtins.lessThan x y</literal>.
-    The comparison operators also work on strings.</para>
-  </listitem>
-
-  <listitem>
-    <para>On 64-bit systems, Nix integers are now 64 bits rather than
-    32 bits.</para>
-  </listitem>
-
-  <listitem>
-    <para>When using the Nix daemon, the <command>nix-daemon</command>
-    worker process now runs on the same CPU as the client, on systems
-    that support setting CPU affinity.  This gives a significant speedup
-    on some systems.</para>
-  </listitem>
-
-  <listitem>
-    <para>If a stack overflow occurs in the Nix evaluator, you now get
-    a proper error message (rather than “Segmentation fault”) on some
-    systems.</para>
-  </listitem>
-
-  <listitem>
-    <para>In addition to directories, you can now bind-mount regular
-    files in chroots through the (now misnamed) option
-    <option>build-chroot-dirs</option>.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from Domen Kožar, Eelco Dolstra,
-Florian Friesdorf, Gergely Risko, Ivan Kozik, Ludovic Courtès and Shea
-Levy.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.7.xml b/doc/manual/release-notes/rl-1.7.xml
deleted file mode 100644
index e736c15a2..000000000
--- a/doc/manual/release-notes/rl-1.7.xml
+++ /dev/null
@@ -1,263 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.7">
-
-<title>Release 1.7 (2014-04-11)</title>
-
-<para>In addition to the usual bug fixes, this release has the
-following new features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Antiquotation is now allowed inside of quoted attribute
-    names (e.g. <literal>set."${foo}"</literal>). In the case where
-    the attribute name is just a single antiquotation, the quotes can
-    be dropped (e.g. the above example can be written
-    <literal>set.${foo}</literal>). If an attribute name inside of a
-    set declaration evaluates to <literal>null</literal> (e.g.
-    <literal>{ ${null} = false; }</literal>), then that attribute is
-    not added to the set.</para>
-  </listitem>
-
-  <listitem>
-    <para>Experimental support for cryptographically signed binary
-    caches.  See <link
-    xlink:href="https://github.com/NixOS/nix/commit/0fdf4da0e979f992db75cc17376e455ddc5a96d8">the
-    commit for details</link>.</para>
-  </listitem>
-
-  <listitem>
-    <para>An experimental new substituter,
-    <command>download-via-ssh</command>, that fetches binaries from
-    remote machines via SSH.  Specifying the flags <literal>--option
-    use-ssh-substituter true --option ssh-substituter-hosts
-    <emphasis>user@hostname</emphasis></literal> will cause Nix
-    to download binaries from the specified machine, if it has
-    them.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-store -r</command> and
-    <command>nix-build</command> have a new flag,
-    <option>--check</option>, that builds a previously built
-    derivation again, and prints an error message if the output is not
-    exactly the same. This helps to verify whether a derivation is
-    truly deterministic.  For example:
-
-<screen>
-$ nix-build '&lt;nixpkgs>' -A patchelf
-<emphasis>…</emphasis>
-$ nix-build '&lt;nixpkgs>' -A patchelf --check
-<emphasis>…</emphasis>
-error: derivation `/nix/store/1ipvxs…-patchelf-0.6' may not be deterministic:
-  hash mismatch in output `/nix/store/4pc1dm…-patchelf-0.6.drv'
-</screen>
-
-    </para>
-
-  </listitem>
-
-  <listitem>
-    <para>The <command>nix-instantiate</command> flags
-    <option>--eval-only</option> and <option>--parse-only</option>
-    have been renamed to <option>--eval</option> and
-    <option>--parse</option>, respectively.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-instantiate</command>,
-    <command>nix-build</command> and <command>nix-shell</command> now
-    have a flag <option>--expr</option> (or <option>-E</option>) that
-    allows you to specify the expression to be evaluated as a command
-    line argument.  For instance, <literal>nix-instantiate --eval -E
-    '1 + 2'</literal> will print <literal>3</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> improvements:</para>
-
-    <itemizedlist>
-
-      <listitem>
-        <para>It has a new flag, <option>--packages</option> (or
-        <option>-p</option>), that sets up a build environment
-        containing the specified packages from Nixpkgs. For example,
-        the command
-
-<screen>
-$ nix-shell -p sqlite xorg.libX11 hello
-</screen>
-
-        will start a shell in which the given packages are
-        present.</para>
-      </listitem>
-
-      <listitem>
-        <para>It now uses <filename>shell.nix</filename> as the
-        default expression, falling back to
-        <filename>default.nix</filename> if the former doesn’t
-        exist.  This makes it convenient to have a
-        <filename>shell.nix</filename> in your project to set up a
-        nice development environment.</para>
-      </listitem>
-
-      <listitem>
-        <para>It evaluates the derivation attribute
-        <varname>shellHook</varname>, if set. Since
-        <literal>stdenv</literal> does not normally execute this hook,
-        it allows you to do <command>nix-shell</command>-specific
-        setup.</para>
-      </listitem>
-
-      <listitem>
-        <para>It preserves the user’s timezone setting.</para>
-      </listitem>
-
-    </itemizedlist>
-
-  </listitem>
-
-  <listitem>
-    <para>In chroots, Nix now sets up a <filename>/dev</filename>
-    containing only a minimal set of devices (such as
-    <filename>/dev/null</filename>). Note that it only does this if
-    you <emphasis>don’t</emphasis> have <filename>/dev</filename>
-    listed in your <option>build-chroot-dirs</option> setting;
-    otherwise, it will bind-mount the <literal>/dev</literal> from
-    outside the chroot.</para>
-
-    <para>Similarly, if you don’t have <filename>/dev/pts</filename> listed
-    in <option>build-chroot-dirs</option>, Nix will mount a private
-    <literal>devpts</literal> filesystem on the chroot’s
-    <filename>/dev/pts</filename>.</para>
-
-  </listitem>
-
-  <listitem>
-    <para>New built-in function: <function>builtins.toJSON</function>,
-    which returns a JSON representation of a value.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env -q</command> has a new flag
-    <option>--json</option> to print a JSON representation of the
-    installed or available packages.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env</command> now supports meta attributes with
-    more complex values, such as attribute sets.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>-A</option> flag now allows attribute names with
-    dots in them, e.g.
-
-<screen>
-$ nix-instantiate --eval '&lt;nixos>' -A 'config.systemd.units."nscd.service".text'
-</screen>
-
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>--max-freed</option> option to
-    <command>nix-store --gc</command> now accepts a unit
-    specifier. For example, <literal>nix-store --gc --max-freed
-    1G</literal> will free up to 1 gigabyte of disk space.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-collect-garbage</command> has a new flag
-    <option>--delete-older-than</option>
-    <emphasis>N</emphasis><literal>d</literal>, which deletes
-    all user environment generations older than
-    <emphasis>N</emphasis> days.  Likewise, <command>nix-env
-    --delete-generations</command> accepts a
-    <emphasis>N</emphasis><literal>d</literal> age limit.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now heuristically detects whether a build failure was
-    due to a disk-full condition. In that case, the build is not
-    flagged as “permanently failed”. This is mostly useful for Hydra,
-    which needs to distinguish between permanent and transient build
-    failures.</para>
-  </listitem>
-
-  <listitem>
-    <para>There is a new symbol <literal>__curPos</literal> that
-    expands to an attribute set containing its file name and line and
-    column numbers, e.g. <literal>{ file = "foo.nix"; line = 10;
-    column = 5; }</literal>.  There also is a new builtin function,
-    <varname>unsafeGetAttrPos</varname>, that returns the position of
-    an attribute.  This is used by Nixpkgs to provide location
-    information in error messages, e.g.
-
-<screen>
-$ nix-build '&lt;nixpkgs>' -A libreoffice --argstr system x86_64-darwin
-error: the package ‘libreoffice-4.0.5.2’ in ‘.../applications/office/libreoffice/default.nix:263’
-  is not supported on ‘x86_64-darwin’
-</screen>
-
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The garbage collector is now more concurrent with other Nix
-    processes because it releases certain locks earlier.</para>
-  </listitem>
-
-  <listitem>
-    <para>The binary tarball installer has been improved.  You can now
-    install Nix by running:
-
-<screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
-</screen>
-
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>More evaluation errors include position information. For
-    instance, selecting a missing attribute will print something like
-
-<screen>
-error: attribute `nixUnstabl' missing, at /etc/nixos/configurations/misc/eelco/mandark.nix:216:15
-</screen>
-
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The command <command>nix-setuid-helper</command> is
-    gone.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix no longer uses Automake, but instead has a
-    non-recursive, GNU Make-based build system.</para>
-  </listitem>
-
-  <listitem>
-    <para>All installed libraries now have the prefix
-    <literal>libnix</literal>.  In particular, this gets rid of
-    <literal>libutil</literal>, which could clash with libraries with
-    the same name from other packages.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now requires a compiler that supports C++11.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from Danny Wilson, Domen Kožar,
-Eelco Dolstra, Ian-Woo Kim, Ludovic Courtès, Maxim Ivanov, Petr
-Rockai, Ricardo M. Correia and Shea Levy.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml
deleted file mode 100644
index d7e2e99ad..000000000
--- a/doc/manual/release-notes/rl-1.8.xml
+++ /dev/null
@@ -1,123 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.8">
-
-<title>Release 1.8 (2014-12-14)</title>
-
-<itemizedlist>
-
-  <listitem><para>Breaking change: to address a race condition, the
-  remote build hook mechanism now uses <command>nix-store
-  --serve</command> on the remote machine. This requires build slaves
-  to be updated to Nix 1.8.</para></listitem>
-
-  <listitem><para>Nix now uses HTTPS instead of HTTP to access the
-  default binary cache,
-  <literal>cache.nixos.org</literal>.</para></listitem>
-
-  <listitem><para><command>nix-env</command> selectors are now regular
-  expressions. For instance, you can do
-
-<screen>
-$ nix-env -qa '.*zip.*'
-</screen>
-
-  to query all packages with a name containing
-  <literal>zip</literal>.</para></listitem>
-
-  <listitem><para><command>nix-store --read-log</command> can now
-  fetch remote build logs. If a build log is not available locally,
-  then ‘nix-store -l’ will now try to download it from the servers
-  listed in the ‘log-servers’ option in nix.conf. For instance, if you
-  have the configuration option
-
-<programlisting>
-log-servers = http://hydra.nixos.org/log
-</programlisting>
-
-then it will try to get logs from
-<literal>http://hydra.nixos.org/log/<emphasis>base name of the
-store path</emphasis></literal>. This allows you to do things like:
-
-<screen>
-$ nix-store -l $(which xterm)
-</screen>
-
-  and get a log even if <command>xterm</command> wasn't built
-  locally.</para></listitem>
-
-  <listitem><para>New builtin functions:
-  <function>attrValues</function>, <function>deepSeq</function>,
-  <function>fromJSON</function>, <function>readDir</function>,
-  <function>seq</function>.</para></listitem>
-
-  <listitem><para><command>nix-instantiate --eval</command> now has a
-  <option>--json</option> flag to print the resulting value in JSON
-  format.</para></listitem>
-
-  <listitem><para><command>nix-copy-closure</command> now uses
-  <command>nix-store --serve</command> on the remote side to send or
-  receive closures. This fixes a race condition between
-  <command>nix-copy-closure</command> and the garbage
-  collector.</para></listitem>
-
-  <listitem><para>Derivations can specify the new special attribute
-  <varname>allowedRequisites</varname>, which has a similar meaning to
-  <varname>allowedReferences</varname>. But instead of only enforcing
-  to explicitly specify the immediate references, it requires the
-  derivation to specify all the dependencies recursively (hence the
-  name, requisites) that are used by the resulting
-  output.</para></listitem>
-
-  <listitem><para>On Mac OS X, Nix now handles case collisions when
-  importing closures from case-sensitive file systems. This is mostly
-  useful for running NixOps on Mac OS X.</para></listitem>
-
-  <listitem><para>The Nix daemon has new configuration options
-  <option>allowed-users</option> (specifying the users and groups that
-  are allowed to connect to the daemon) and
-  <option>trusted-users</option> (specifying the users and groups that
-  can perform privileged operations like specifying untrusted binary
-  caches).</para></listitem>
-
-  <listitem><para>The configuration option
-  <option>build-cores</option> now defaults to the number of available
-  CPU cores.</para></listitem>
-
-  <listitem><para>Build users are now used by default when Nix is
-  invoked as root. This prevents builds from accidentally running as
-  root.</para></listitem>
-
-  <listitem><para>Nix now includes systemd units and Upstart
-  jobs.</para></listitem>
-
-  <listitem><para>Speed improvements to <command>nix-store
-  --optimise</command>.</para></listitem>
-
-  <listitem><para>Language change: the <literal>==</literal> operator
-  now ignores string contexts (the “dependencies” of a
-  string).</para></listitem>
-
-  <listitem><para>Nix now filters out Nix-specific ANSI escape
-  sequences on standard error. They are supposed to be invisible, but
-  some terminals show them anyway.</para></listitem>
-
-  <listitem><para>Various commands now automatically pipe their output
-  into the pager as specified by the <literal>PAGER</literal> environment
-  variable.</para></listitem>
-
-  <listitem><para>Several improvements to reduce memory consumption in
-  the evaluator.</para></listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from Adam Szkoda, Aristid
-Breitkreuz, Bob van der Linden, Charles Strahan, darealshinji, Eelco
-Dolstra, Gergely Risko, Joel Taylor, Ludovic Courtès, Marko Durkovic,
-Mikey Ariel, Paul Colomiets, Ricardo M.  Correia, Ricky Elrod, Robert
-Helgesson, Rob Vermaas, Russell O'Connor, Shea Levy, Shell Turner,
-Sönke Hahn, Steve Purcell, Vladimír Čunát and Wout Mertens.</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-1.9.xml b/doc/manual/release-notes/rl-1.9.xml
deleted file mode 100644
index cc6558b1a..000000000
--- a/doc/manual/release-notes/rl-1.9.xml
+++ /dev/null
@@ -1,216 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-1.9">
-
-<title>Release 1.9 (2015-06-12)</title>
-
-<para>In addition to the usual bug fixes, this release has the
-following new features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Signed binary cache support. You can enable signature
-    checking by adding the following to <filename>nix.conf</filename>:
-
-<programlisting>
-signed-binary-caches = *
-binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-</programlisting>
-
-    This will prevent Nix from downloading any binary from the cache
-    that is not signed by one of the keys listed in
-    <option>binary-cache-public-keys</option>.</para>
-
-    <para>Signature checking is only supported if you built Nix with
-    the <literal>libsodium</literal> package.</para>
-
-    <para>Note that while Nix has had experimental support for signed
-    binary caches since version 1.7, this release changes the
-    signature format in a backwards-incompatible way.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Automatic downloading of Nix expression tarballs. In various
-    places, you can now specify the URL of a tarball containing Nix
-    expressions (such as Nixpkgs), which will be downloaded and
-    unpacked automatically. For example:</para>
-
-    <itemizedlist>
-
-      <listitem><para>In <command>nix-env</command>:
-
-<screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
-</screen>
-
-      This installs Firefox from the latest tested and built revision
-      of the NixOS 14.12 channel.</para></listitem>
-
-      <listitem><para>In <command>nix-build</command> and
-      <command>nix-shell</command>:
-
-<screen>
-$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
-</screen>
-
-      This builds GNU Hello from the latest revision of the Nixpkgs
-      master branch.</para></listitem>
-
-      <listitem><para>In the Nix search path (as specified via
-      <literal>NIX_PATH</literal> or <option>-I</option>). For example, to
-      start a shell containing the Pan package from a specific version
-      of Nixpkgs:
-
-<screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
-</screen>
-
-      </para></listitem>
-
-      <listitem><para>In <command>nixos-rebuild</command> (on NixOS):
-
-<screen>
-$ nixos-rebuild test -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz
-</screen>
-
-      </para></listitem>
-
-      <listitem><para>In Nix expressions, via the new builtin function <function>fetchTarball</function>:
-
-<programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; …
-</programlisting>
-
-      (This is not allowed in restricted mode.)</para></listitem>
-
-    </itemizedlist>
-
-  </listitem>
-
-  <listitem>
-
-    <para><command>nix-shell</command> improvements:</para>
-
-    <itemizedlist>
-
-      <listitem><para><command>nix-shell</command> now has a flag
-      <option>--run</option> to execute a command in the
-      <command>nix-shell</command> environment,
-      e.g. <literal>nix-shell --run make</literal>. This is like
-      the existing <option>--command</option> flag, except that it
-      uses a non-interactive shell (ensuring that hitting Ctrl-C won’t
-      drop you into the child shell).</para></listitem>
-
-      <listitem><para><command>nix-shell</command> can now be used as
-      a <literal>#!</literal>-interpreter. This allows you to write
-      scripts that dynamically fetch their own dependencies. For
-      example, here is a Haskell script that, when invoked, first
-      downloads GHC and the Haskell packages on which it depends:
-
-<programlisting>
-#! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP
-
-import Network.HTTP
-
-main = do
-  resp &lt;- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
-  body &lt;- getResponseBody resp
-  print (take 100 body)
-</programlisting>
-
-      Of course, the dependencies are cached in the Nix store, so the
-      second invocation of this script will be much
-      faster.</para></listitem>
-
-    </itemizedlist>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Chroot improvements:</para>
-
-    <itemizedlist>
-
-      <listitem><para>Chroot builds are now supported on Mac OS X
-      (using its sandbox mechanism).</para></listitem>
-
-      <listitem><para>If chroots are enabled, they are now used for
-      all derivations, including fixed-output derivations (such as
-      <function>fetchurl</function>). The latter do have network
-      access, but can no longer access the host filesystem. If you
-      need the old behaviour, you can set the option
-      <option>build-use-chroot</option> to
-      <literal>relaxed</literal>.</para></listitem>
-
-      <listitem><para>On Linux, if chroots are enabled, builds are
-      performed in a private PID namespace once again. (This
-      functionality was lost in Nix 1.8.)</para></listitem>
-
-      <listitem><para>Store paths listed in
-      <option>build-chroot-dirs</option> are now automatically
-      expanded to their closure. For instance, if you want
-      <filename>/nix/store/…-bash/bin/sh</filename> mounted in your
-      chroot as <filename>/bin/sh</filename>, you only need to say
-      <literal>build-chroot-dirs =
-      /bin/sh=/nix/store/…-bash/bin/sh</literal>; it is no longer
-      necessary to specify the dependencies of Bash.</para></listitem>
-
-    </itemizedlist>
-
-  </listitem>
-
-  <listitem><para>The new derivation attribute
-  <varname>passAsFile</varname> allows you to specify that the
-  contents of derivation attributes should be passed via files rather
-  than environment variables. This is useful if you need to pass very
-  long strings that exceed the size limit of the environment. The
-  Nixpkgs function <function>writeTextFile</function> uses
-  this.</para></listitem>
-
-  <listitem><para>You can now use <literal>~</literal> in Nix file
-  names to refer to your home directory, e.g. <literal>import
-  ~/.nixpkgs/config.nix</literal>.</para></listitem>
-
-  <listitem><para>Nix has a new option <option>restrict-eval</option>
-  that allows limiting what paths the Nix evaluator has access to. By
-  passing <literal>--option restrict-eval true</literal> to Nix, the
-  evaluator will throw an exception if an attempt is made to access
-  any file outside of the Nix search path. This is primarily intended
-  for Hydra to ensure that a Hydra jobset only refers to its declared
-  inputs (and is therefore reproducible).</para></listitem>
-
-  <listitem><para><command>nix-env</command> now only creates a new
-  “generation” symlink in <filename>/nix/var/nix/profiles</filename>
-  if something actually changed.</para></listitem>
-
-  <listitem><para>The environment variable <literal>NIX_PAGER</literal>
-  can now be set to override <literal>PAGER</literal>. You can set it to
-  <literal>cat</literal> to disable paging for Nix commands
-  only.</para></listitem>
-
-  <listitem><para>Failing <literal>&lt;...></literal>
-  lookups now show position information.</para></listitem>
-
-  <listitem><para>Improved Boehm GC use: we disabled scanning for
-  interior pointers, which should reduce the “<literal>Repeated
-  allocation of very large block</literal>” warnings and associated
-  retention of memory.</para></listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from aszlig, Benjamin Staffin,
-Charles Strahan, Christian Theune, Daniel Hahler, Danylo Hlynskyi
-Daniel Peebles, Dan Peebles, Domen Kožar, Eelco Dolstra, Harald van
-Dijk, Hoang Xuan Phu, Jaka Hudoklin, Jeff Ramnani, j-keck, Linquize,
-Luca Bruno, Michael Merickel, Oliver Dunkl, Rob Vermaas, Rok Garbas,
-Shea Levy, Tobias Geerinckx-Rice and William A. Kennington III.</para>
-
-</section>
-
diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
deleted file mode 100644
index 6bef003ca..000000000
--- a/doc/manual/release-notes/rl-2.0.xml
+++ /dev/null
@@ -1,1012 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-2.0">
-
-<title>Release 2.0 (2018-02-22)</title>
-
-<para>The following incompatible changes have been made:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>The manifest-based substituter mechanism
-    (<command>download-using-manifests</command>) has been <link
-    xlink:href="https://github.com/NixOS/nix/commit/867967265b80946dfe1db72d40324b4f9af988ed">removed</link>. It
-    has been superseded by the binary cache substituter mechanism
-    since several years. As a result, the following programs have been
-    removed:
-
-    <itemizedlist>
-      <listitem><para><command>nix-pull</command></para></listitem>
-      <listitem><para><command>nix-generate-patches</command></para></listitem>
-      <listitem><para><command>bsdiff</command></para></listitem>
-      <listitem><para><command>bspatch</command></para></listitem>
-    </itemizedlist>
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The “copy from other stores” substituter mechanism
-    (<command>copy-from-other-stores</command> and the
-    <literal>NIX_OTHER_STORES</literal> environment variable) has been
-    removed. It was primarily used by the NixOS installer to copy
-    available paths from the installation medium. The replacement is
-    to use a chroot store as a substituter
-    (e.g. <literal>--substituters /mnt</literal>), or to build into a
-    chroot store (e.g. <literal>--store /mnt --substituters /</literal>).</para>
-  </listitem>
-
-  <listitem>
-    <para>The command <command>nix-push</command> has been removed as
-    part of the effort to eliminate Nix's dependency on Perl. You can
-    use <command>nix copy</command> instead, e.g. <literal>nix copy
-    --to file:///tmp/my-binary-cache <emphasis>paths…</emphasis></literal></para>
-  </listitem>
-
-  <listitem>
-    <para>The “nested” log output feature (<option>--log-type
-    pretty</option>) has been removed. As a result,
-    <command>nix-log2xml</command> was also removed.</para>
-  </listitem>
-
-  <listitem>
-    <para>OpenSSL-based signing has been <link
-    xlink:href="https://github.com/NixOS/nix/commit/f435f8247553656774dd1b2c88e9de5d59cab203">removed</link>. This
-    feature was never well-supported. A better alternative is provided
-    by the <option>secret-key-files</option> and
-    <option>trusted-public-keys</option> options.</para>
-  </listitem>
-
-  <listitem>
-    <para>Failed build caching has been <link
-    xlink:href="https://github.com/NixOS/nix/commit/8cffec84859cec8b610a2a22ab0c4d462a9351ff">removed</link>. This
-    feature was introduced to support the Hydra continuous build
-    system, but Hydra no longer uses it.</para>
-  </listitem>
-
-  <listitem>
-    <para><filename>nix-mode.el</filename> has been removed from
-    Nix. It is now <link
-    xlink:href="https://github.com/NixOS/nix-mode">a separate
-    repository</link> and can be installed through the MELPA package
-    repository.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has the following new features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>It introduces a new command named <command>nix</command>,
-    which is intended to eventually replace all
-    <command>nix-*</command> commands with a more consistent and
-    better designed user interface. It currently provides replacements
-    for some (but not all) of the functionality provided by
-    <command>nix-store</command>, <command>nix-build</command>,
-    <command>nix-shell -p</command>, <command>nix-env -qa</command>,
-    <command>nix-instantiate --eval</command>,
-    <command>nix-push</command> and
-    <command>nix-copy-closure</command>. It has the following major
-    features:</para>
-
-    <itemizedlist>
-
-      <listitem>
-        <para>Unlike the legacy commands, it has a consistent way to
-        refer to packages and package-like arguments (like store
-        paths). For example, the following commands all copy the GNU
-        Hello package to a remote machine:
-
-        <screen>nix copy --to ssh://machine nixpkgs.hello</screen>
-        <screen>nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen>
-        <screen>nix copy --to ssh://machine '(with import &lt;nixpkgs> {}; hello)'</screen>
-
-        By contrast, <command>nix-copy-closure</command> only accepted
-        store paths as arguments.</para>
-      </listitem>
-
-      <listitem>
-        <para>It is self-documenting: <option>--help</option> shows
-        all available command-line arguments. If
-        <option>--help</option> is given after a subcommand, it shows
-        examples for that subcommand. <command>nix
-        --help-config</command> shows all configuration
-        options.</para>
-      </listitem>
-
-      <listitem>
-        <para>It is much less verbose. By default, it displays a
-        single-line progress indicator that shows how many packages
-        are left to be built or downloaded, and (if there are running
-        builds) the most recent line of builder output. If a build
-        fails, it shows the last few lines of builder output. The full
-        build log can be retrieved using <command>nix
-        log</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para>It <link
-        xlink:href="https://github.com/NixOS/nix/commit/b8283773bd64d7da6859ed520ee19867742a03ba">provides</link>
-        all <filename>nix.conf</filename> configuration options as
-        command line flags. For example, instead of <literal>--option
-        http-connections 100</literal> you can write
-        <literal>--http-connections 100</literal>. Boolean options can
-        be written as
-        <literal>--<emphasis>foo</emphasis></literal> or
-        <literal>--no-<emphasis>foo</emphasis></literal>
-        (e.g. <option>--no-auto-optimise-store</option>).</para>
-      </listitem>
-
-      <listitem>
-        <para>Many subcommands have a <option>--json</option> flag to
-        write results to stdout in JSON format.</para>
-      </listitem>
-
-    </itemizedlist>
-
-    <warning><para>Please note that the <command>nix</command> command
-    is a work in progress and the interface is subject to
-    change.</para></warning>
-
-    <para>It provides the following high-level (“porcelain”)
-    subcommands:</para>
-
-    <itemizedlist>
-
-      <listitem>
-        <para><command>nix build</command> is a replacement for
-        <command>nix-build</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix run</command> executes a command in an
-        environment in which the specified packages are available. It
-        is (roughly) a replacement for <command>nix-shell
-        -p</command>. Unlike that command, it does not execute the
-        command in a shell, and has a flag (<command>-c</command>)
-        that specifies the unquoted command line to be
-        executed.</para>
-
-        <para>It is particularly useful in conjunction with chroot
-        stores, allowing Linux users who do not have permission to
-        install Nix in <command>/nix/store</command> to still use
-        binary substitutes that assume
-        <command>/nix/store</command>. For example,
-
-        <screen>nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!'</screen>
-
-        downloads (or if not substitutes are available, builds) the
-        GNU Hello package into
-        <filename>~/my-nix/nix/store</filename>, then runs
-        <command>hello</command> in a mount namespace where
-        <filename>~/my-nix/nix/store</filename> is mounted onto
-        <command>/nix/store</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix search</command> replaces <command>nix-env
-        -qa</command>. It searches the available packages for
-        occurrences of a search string in the attribute name, package
-        name or description. Unlike <command>nix-env -qa</command>, it
-        has a cache to speed up subsequent searches.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix copy</command> copies paths between
-        arbitrary Nix stores, generalising
-        <command>nix-copy-closure</command> and
-        <command>nix-push</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix repl</command> replaces the external
-        program <command>nix-repl</command>. It provides an
-        interactive environment for evaluating and building Nix
-        expressions. Note that it uses <literal>linenoise-ng</literal>
-        instead of GNU Readline.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix upgrade-nix</command> upgrades Nix to the
-        latest stable version. This requires that Nix is installed in
-        a profile. (Thus it won’t work on NixOS, or if it’s installed
-        outside of the Nix store.)</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix verify</command> checks whether store paths
-        are unmodified and/or “trusted” (see below). It replaces
-        <command>nix-store --verify</command> and <command>nix-store
-        --verify-path</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix log</command> shows the build log of a
-        package or path. If the build log is not available locally, it
-        will try to obtain it from the configured substituters (such
-        as <uri>cache.nixos.org</uri>, which now provides build
-        logs).</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix edit</command> opens the source code of a
-        package in your editor.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix eval</command> replaces
-        <command>nix-instantiate --eval</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command
-        xlink:href="https://github.com/NixOS/nix/commit/d41c5eb13f4f3a37d80dbc6d3888644170c3b44a">nix
-        why-depends</command> shows why one store path has another in
-        its closure. This is primarily useful to finding the causes of
-        closure bloat. For example,
-
-        <screen>nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev</screen>
-
-        shows a chain of files and fragments of file contents that
-        cause the VLC package to have the “dev” output of
-        <literal>libdrm</literal> in its closure — an undesirable
-        situation.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix path-info</command> shows information about
-        store paths, replacing <command>nix-store -q</command>. A
-        useful feature is the option <option>--closure-size</option>
-        (<option>-S</option>). For example, the following command show
-        the closure sizes of every path in the current NixOS system
-        closure, sorted by size:
-
-        <screen>nix path-info -rS /run/current-system | sort -nk2</screen>
-
-        </para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix optimise-store</command> replaces
-        <command>nix-store --optimise</command>. The main difference
-        is that it has a progress indicator.</para>
-      </listitem>
-
-    </itemizedlist>
-
-    <para>A number of low-level (“plumbing”) commands are also
-    available:</para>
-
-    <itemizedlist>
-
-      <listitem>
-        <para><command>nix ls-store</command> and <command>nix
-        ls-nar</command> list the contents of a store path or NAR
-        file. The former is primarily useful in conjunction with
-        remote stores, e.g.
-
-        <screen>nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen>
-
-        lists the contents of path in a binary cache.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix cat-store</command> and <command>nix
-        cat-nar</command> allow extracting a file from a store path or
-        NAR file.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix dump-path</command> writes the contents of
-        a store path to stdout in NAR format. This replaces
-        <command>nix-store --dump</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command
-        xlink:href="https://github.com/NixOS/nix/commit/e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2">nix
-        show-derivation</command> displays a store derivation in JSON
-        format. This is an alternative to
-        <command>pp-aterm</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command
-        xlink:href="https://github.com/NixOS/nix/commit/970366266b8df712f5f9cedb45af183ef5a8357f">nix
-        add-to-store</command> replaces <command>nix-store
-        --add</command>.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix sign-paths</command> signs store
-        paths.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix copy-sigs</command> copies signatures from
-        one store to another.</para>
-      </listitem>
-
-      <listitem>
-        <para><command>nix show-config</command> shows all
-        configuration options and their current values.</para>
-      </listitem>
-
-    </itemizedlist>
-
-  </listitem>
-
-  <listitem>
-    <para>The store abstraction that Nix has had for a long time to
-    support store access via the Nix daemon has been extended
-    significantly. In particular, substituters (which used to be
-    external programs such as
-    <command>download-from-binary-cache</command>) are now subclasses
-    of the abstract <classname>Store</classname> class. This allows
-    many Nix commands to operate on such store types. For example,
-    <command>nix path-info</command> shows information about paths in
-    your local Nix store, while <command>nix path-info --store
-    https://cache.nixos.org/</command> shows information about paths
-    in the specified binary cache. Similarly,
-    <command>nix-copy-closure</command>, <command>nix-push</command>
-    and substitution are all instances of the general notion of
-    copying paths between different kinds of Nix stores.</para>
-
-    <para>Stores are specified using an URI-like syntax,
-    e.g. <uri>https://cache.nixos.org/</uri> or
-    <uri>ssh://machine</uri>. The following store types are supported:
-
-    <itemizedlist>
-
-      <listitem>
-
-        <para><classname>LocalStore</classname> (stori URI
-        <literal>local</literal> or an absolute path) and the misnamed
-        <classname>RemoteStore</classname> (<literal>daemon</literal>)
-        provide access to a local Nix store, the latter via the Nix
-        daemon. You can use <literal>auto</literal> or the empty
-        string to auto-select a local or daemon store depending on
-        whether you have write permission to the Nix store. It is no
-        longer necessary to set the <literal>NIX_REMOTE</literal>
-        environment variable to use the Nix daemon.</para>
-
-        <para>As noted above, <classname>LocalStore</classname> now
-        supports chroot builds, allowing the “physical” location of
-        the Nix store
-        (e.g. <filename>/home/alice/nix/store</filename>) to differ
-        from its “logical” location (typically
-        <filename>/nix/store</filename>). This allows non-root users
-        to use Nix while still getting the benefits from prebuilt
-        binaries from <uri>cache.nixos.org</uri>.</para>
-
-      </listitem>
-
-      <listitem>
-
-        <para><classname>BinaryCacheStore</classname> is the abstract
-        superclass of all binary cache stores. It supports writing
-        build logs and NAR content listings in JSON format.</para>
-
-      </listitem>
-
-      <listitem>
-
-        <para><classname>HttpBinaryCacheStore</classname>
-        (<literal>http://</literal>, <literal>https://</literal>)
-        supports binary caches via HTTP or HTTPS. If the server
-        supports <literal>PUT</literal> requests, it supports
-        uploading store paths via commands such as <command>nix
-        copy</command>.</para>
-
-      </listitem>
-
-      <listitem>
-
-        <para><classname>LocalBinaryCacheStore</classname>
-        (<literal>file://</literal>) supports binary caches in the
-        local filesystem.</para>
-
-      </listitem>
-
-      <listitem>
-
-        <para><classname>S3BinaryCacheStore</classname>
-        (<literal>s3://</literal>) supports binary caches stored in
-        Amazon S3, if enabled at compile time.</para>
-
-      </listitem>
-
-      <listitem>
-
-        <para><classname>LegacySSHStore</classname> (<literal>ssh://</literal>)
-        is used to implement remote builds and
-        <command>nix-copy-closure</command>.</para>
-
-      </listitem>
-
-      <listitem>
-
-        <para><classname>SSHStore</classname>
-        (<literal>ssh-ng://</literal>) supports arbitrary Nix
-        operations on a remote machine via the same protocol used by
-        <command>nix-daemon</command>.</para>
-
-      </listitem>
-
-    </itemizedlist>
-
-    </para>
-
-  </listitem>
-
-  <listitem>
-
-    <para>Security has been improved in various ways:
-
-    <itemizedlist>
-
-      <listitem>
-        <para>Nix now stores signatures for local store
-        paths. When paths are copied between stores (e.g., copied from
-        a binary cache to a local store), signatures are
-        propagated.</para>
-
-        <para>Locally-built paths are signed automatically using the
-        secret keys specified by the <option>secret-key-files</option>
-        store option. Secret/public key pairs can be generated using
-        <command>nix-store
-        --generate-binary-cache-key</command>.</para>
-
-        <para>In addition, locally-built store paths are marked as
-        “ultimately trusted”, but this bit is not propagated when
-        paths are copied between stores.</para>
-      </listitem>
-
-      <listitem>
-        <para>Content-addressable store paths no longer require
-        signatures — they can be imported into a store by unprivileged
-        users even if they lack signatures.</para>
-      </listitem>
-
-      <listitem>
-        <para>The command <command>nix verify</command> checks whether
-        the specified paths are trusted, i.e., have a certain number
-        of trusted signatures, are ultimately trusted, or are
-        content-addressed.</para>
-      </listitem>
-
-      <listitem>
-        <para>Substitutions from binary caches <link
-        xlink:href="https://github.com/NixOS/nix/commit/ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b">now</link>
-        require signatures by default. This was already the case on
-        NixOS.</para>
-      </listitem>
-
-      <listitem>
-        <para>In Linux sandbox builds, we <link
-        xlink:href="https://github.com/NixOS/nix/commit/eba840c8a13b465ace90172ff76a0db2899ab11b">now</link>
-        use <filename>/build</filename> instead of
-        <filename>/tmp</filename> as the temporary build
-        directory. This fixes potential security problems when a build
-        accidentally stores its <literal>TMPDIR</literal> in some
-        security-sensitive place, such as an RPATH.</para>
-      </listitem>
-
-    </itemizedlist>
-
-    </para>
-
-  </listitem>
-
-  <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. With the
-    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
-    restricted evaluation mode that forbids access to anything that could cause
-    different evaluations of the same command line arguments to produce a
-    different result. This includes builtin functions such as
-    <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a content hash
-    or commit hash is specified. For example, calls to
-    <function>builtins.fetchGit</function> are only allowed if a
-    <varname>rev</varname> attribute is specified.</para>
-
-    <para>The goal of this feature is to enable true reproducibility
-    and traceability of builds (including NixOS system configurations)
-    at the evaluation level. For example, in the future,
-    <command>nixos-rebuild</command> might build configurations from a
-    Nix expression in a Git repository in pure mode. That expression
-    might fetch other repositories such as Nixpkgs via
-    <function>builtins.fetchGit</function>. The commit hash of the
-    top-level repository then uniquely identifies a running system,
-    and, in conjunction with that repository, allows it to be
-    reproduced or modified.</para>
-
-  </listitem>
-
-  <listitem>
-    <para>There are several new features to support binary
-    reproducibility (i.e. to help ensure that multiple builds of the
-    same derivation produce exactly the same output). When
-    <option>enforce-determinism</option> is set to
-    <literal>false</literal>, it’s <link
-    xlink:href="https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051">no
-    longer</link> a fatal error if build rounds produce different
-    output. Also, a hook named <option>diff-hook</option> is <link
-    xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169">provided</link>
-    to allow you to run tools such as <command>diffoscope</command>
-    when build rounds produce different output.</para>
-  </listitem>
-
-  <listitem>
-    <para>Configuring remote builds is a lot easier now. Provided you
-    are not using the Nix daemon, you can now just specify a remote
-    build machine on the command line, e.g. <literal>--option builders
-    'ssh://my-mac x86_64-darwin'</literal>. The environment variable
-    <literal>NIX_BUILD_HOOK</literal> has been removed and is no longer
-    needed. The environment variable <literal>NIX_REMOTE_SYSTEMS</literal>
-    is still supported for compatibility, but it is also possible to
-    specify builders in <command>nix.conf</command> by setting the
-    option <literal>builders =
-    @<emphasis>path</emphasis></literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para>If a fixed-output derivation produces a result with an
-    incorrect hash, the output path is moved to the location
-    corresponding to the actual hash and registered as valid. Thus, a
-    subsequent build of the fixed-output derivation with the correct
-    hash is unnecessary.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> <link
-    xlink:href="https://github.com/NixOS/nix/commit/ea59f39326c8e9dc42dfed4bcbf597fbce58797c">now</link>
-    sets the <varname>IN_NIX_SHELL</varname> environment variable
-    during evaluation and in the shell itself. This can be used to
-    perform different actions depending on whether you’re in a Nix
-    shell or in a regular build. Nixpkgs provides
-    <varname>lib.inNixShell</varname> to check this variable during
-    evaluation.</para>
-  </listitem>
-
-  <listitem>
-    <para><literal>NIX_PATH</literal> is now lazy, so URIs in the path are
-    only downloaded if they are needed for evaluation.</para>
-  </listitem>
-
-  <listitem>
-    <para>You can now use
-    <uri>channel:<emphasis>channel-name</emphasis></uri> as a
-    short-hand for
-    <uri>https://nixos.org/channels/<emphasis>channel-name</emphasis>/nixexprs.tar.xz</uri>. For
-    example, <literal>nix-build channel:nixos-15.09 -A hello</literal>
-    will build the GNU Hello package from the
-    <literal>nixos-15.09</literal> channel. In the future, this may
-    use Git to fetch updates more efficiently.</para>
-  </listitem>
-
-  <listitem>
-    <para>When <option>--no-build-output</option> is given, the last
-    10 lines of the build log will be shown if a build
-    fails.</para>
-  </listitem>
-
-  <listitem>
-    <para>Networking has been improved:
-
-    <itemizedlist>
-
-      <listitem>
-        <para>HTTP/2 is now supported. This makes binary cache lookups
-        <link
-        xlink:href="https://github.com/NixOS/nix/commit/90ad02bf626b885a5dd8967894e2eafc953bdf92">much
-        more efficient</link>.</para>
-      </listitem>
-
-      <listitem>
-        <para>We now retry downloads on many HTTP errors, making
-        binary caches substituters more resilient to temporary
-        failures.</para>
-      </listitem>
-
-      <listitem>
-        <para>HTTP credentials can now be configured via the standard
-        <filename>netrc</filename> mechanism.</para>
-      </listitem>
-
-      <listitem>
-        <para>If S3 support is enabled at compile time,
-        <uri>s3://</uri> URIs are <link
-        xlink:href="https://github.com/NixOS/nix/commit/9ff9c3f2f80ba4108e9c945bbfda2c64735f987b">supported</link>
-        in all places where Nix allows URIs.</para>
-      </listitem>
-
-      <listitem>
-        <para>Brotli compression is now supported. In particular,
-        <uri>cache.nixos.org</uri> build logs are now compressed using
-        Brotli.</para>
-      </listitem>
-
-    </itemizedlist>
-
-    </para>
-
-  </listitem>
-
-  <listitem>
-    <para><command>nix-env</command> <link
-    xlink:href="https://github.com/NixOS/nix/commit/b0cb11722626e906a73f10dd9a0c9eea29faf43a">now</link>
-    ignores packages with bad derivation names (in particular those
-    starting with a digit or containing a dot).</para>
-  </listitem>
-
-  <listitem>
-    <para>Many configuration options have been renamed, either because
-    they were unnecessarily verbose
-    (e.g. <option>build-use-sandbox</option> is now just
-    <option>sandbox</option>) or to reflect generalised behaviour
-    (e.g. <option>binary-caches</option> is now
-    <option>substituters</option> because it allows arbitrary store
-    URIs). The old names are still supported for compatibility.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>max-jobs</option> option can <link
-    xlink:href="https://github.com/NixOS/nix/commit/7251d048fa812d2551b7003bc9f13a8f5d4c95a5">now</link>
-    be set to <literal>auto</literal> to use the number of CPUs in the
-    system.</para>
-  </listitem>
-
-  <listitem>
-    <para>Hashes can <link
-    xlink:href="https://github.com/NixOS/nix/commit/c0015e87af70f539f24d2aa2bc224a9d8b84276b">now</link>
-    be specified in base-64 format, in addition to base-16 and the
-    non-standard base-32.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-shell</command> now uses
-    <varname>bashInteractive</varname> from Nixpkgs, rather than the
-    <command>bash</command> command that happens to be in the caller’s
-    <literal>PATH</literal>. This is especially important on macOS where
-    the <command>bash</command> provided by the system is seriously
-    outdated and cannot execute <literal>stdenv</literal>’s setup
-    script.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix can now automatically trigger a garbage collection if
-    free disk space drops below a certain level during a build. This
-    is configured using the <option>min-free</option> and
-    <option>max-free</option> options.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-store -q --roots</command> and
-    <command>nix-store --gc --print-roots</command> now show temporary
-    and in-memory roots.</para>
-  </listitem>
-
-  <listitem>
-    <para>
-      Nix can now be extended with plugins. See the documentation of
-      the <option>plugin-files</option> option for more details.
-    </para>
-  </listitem>
-
-</itemizedlist>
-
-<para>The Nix language has the following new features:
-
-<itemizedlist>
-
-  <listitem>
-    <para>It supports floating point numbers. They are based on the
-    C++ <literal>float</literal> type and are supported by the
-    existing numerical operators. Export and import to and from JSON
-    and XML works, too.</para>
-  </listitem>
-
-  <listitem>
-    <para>Derivation attributes can now reference the outputs of the
-    derivation using the <function>placeholder</function> builtin
-    function. For example, the attribute
-
-<programlisting>
-configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";
-</programlisting>
-
-    will cause the <literal>configureFlags</literal> environment variable
-    to contain the actual store paths corresponding to the
-    <literal>out</literal> and <literal>dev</literal> outputs.</para>
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>The following builtin functions are new or extended:
-
-<itemizedlist>
-
-  <listitem>
-    <para><function
-    xlink:href="https://github.com/NixOS/nix/commit/38539b943a060d9cdfc24d6e5d997c0885b8aa2f">builtins.fetchGit</function>
-    allows Git repositories to be fetched at evaluation time. Thus it
-    differs from the <function>fetchgit</function> function in
-    Nixpkgs, which fetches at build time and cannot be used to fetch
-    Nix expressions during evaluation. A typical use case is to import
-    external NixOS modules from your configuration, e.g.
-
-    <programlisting>imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ];</programlisting>
-
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>Similarly, <function>builtins.fetchMercurial</function>
-    allows you to fetch Mercurial repositories.</para>
-  </listitem>
-
-  <listitem>
-    <para><function>builtins.path</function> generalises
-    <function>builtins.filterSource</function> and path literals
-    (e.g. <literal>./foo</literal>). It allows specifying a store path
-    name that differs from the source path name
-    (e.g. <literal>builtins.path { path = ./foo; name = "bar";
-    }</literal>) and also supports filtering out unwanted
-    files.</para>
-  </listitem>
-
-  <listitem>
-    <para><function>builtins.fetchurl</function> and
-    <function>builtins.fetchTarball</function> now support
-    <varname>sha256</varname> and <varname>name</varname>
-    attributes.</para>
-  </listitem>
-
-  <listitem>
-    <para><function
-    xlink:href="https://github.com/NixOS/nix/commit/b8867a0239b1930a16f9ef3f7f3e864b01416dff">builtins.split</function>
-    splits a string using a POSIX extended regular expression as the
-    separator.</para>
-  </listitem>
-
-  <listitem>
-    <para><function
-    xlink:href="https://github.com/NixOS/nix/commit/26d92017d3b36cff940dcb7d1611c42232edb81a">builtins.partition</function>
-    partitions the elements of a list into two lists, depending on a
-    Boolean predicate.</para>
-  </listitem>
-
-  <listitem>
-    <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
-    content-addressable tarball cache at
-    <uri>http://tarballs.nixos.org/</uri>, just like
-    <function>fetchurl</function> in
-    Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para>
-  </listitem>
-
-  <listitem>
-    <para>In restricted and pure evaluation mode, builtin functions
-    that download from the network (such as
-    <function>fetchGit</function>) are permitted to fetch underneath a
-    list of URI prefixes specified in the option
-    <option>allowed-uris</option>.</para>
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>The Nix build environment has the following changes:
-
-<itemizedlist>
-
-  <listitem>
-    <para>Values such as Booleans, integers, (nested) lists and
-    attribute sets can <link
-    xlink:href="https://github.com/NixOS/nix/commit/6de33a9c675b187437a2e1abbcb290981a89ecb1">now</link>
-    be passed to builders in a non-lossy way. If the special attribute
-    <varname>__structuredAttrs</varname> is set to
-    <literal>true</literal>, the other derivation attributes are
-    serialised in JSON format and made available to the builder via
-    the file <literal>.attrs.json</literal> in the builder’s temporary
-    directory. This obviates the need for
-    <varname>passAsFile</varname> since JSON files have no size
-    restrictions, unlike process environments.</para>
-
-    <para><link
-    xlink:href="https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699">As
-    a convenience to Bash builders</link>, Nix writes a script named
-    <literal>.attrs.sh</literal> to the builder’s directory that
-    initialises shell variables corresponding to all attributes that
-    are representable in Bash. This includes non-nested (associative)
-    arrays. For example, the attribute <literal>hardening.format =
-    true</literal> ends up as the Bash associative array element
-    <literal>${hardening[format]}</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para>Builders can <link
-    xlink:href="https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3">now</link>
-    communicate what build phase they are in by writing messages to
-    the file descriptor specified in <literal>NIX_LOG_FD</literal>. The
-    current phase is shown by the <command>nix</command> progress
-    indicator.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>In Linux sandbox builds, we <link
-    xlink:href="https://github.com/NixOS/nix/commit/a2d92bb20e82a0957067ede60e91fab256948b41">now</link>
-    provide a default <filename>/bin/sh</filename> (namely
-    <filename>ash</filename> from BusyBox).</para>
-  </listitem>
-
-  <listitem>
-    <para>In structured attribute mode,
-    <varname>exportReferencesGraph</varname> <link
-    xlink:href="https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a">exports</link>
-    extended information about closures in JSON format. In particular,
-    it includes the sizes and hashes of paths. This is primarily
-    useful for NixOS image builders.</para>
-  </listitem>
-
-  <listitem>
-    <para>Builds are <link
-    xlink:href="https://github.com/NixOS/nix/commit/21948deed99a3295e4d5666e027a6ca42dc00b40">now</link>
-    killed as soon as Nix receives EOF on the builder’s stdout or
-    stderr. This fixes a bug that allowed builds to hang Nix
-    indefinitely, regardless of
-    timeouts.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <option>sandbox-paths</option> configuration
-    option can now specify optional paths by appending a
-    <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will
-    bind-mount <varname>/dev/nvidiactl</varname> only if it
-    exists.</para>
-  </listitem>
-
-  <listitem>
-    <para>On Linux, builds are now executed in a user
-    namespace with UID 1000 and GID 100.</para>
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>A number of significant internal changes were made:
-
-<itemizedlist>
-
-  <listitem>
-    <para>Nix no longer depends on Perl and all Perl components have
-    been rewritten in C++ or removed. The Perl bindings that used to
-    be part of Nix have been moved to a separate package,
-    <literal>nix-perl</literal>.</para>
-  </listitem>
-
-  <listitem>
-    <para>All <classname>Store</classname> classes are now
-    thread-safe. <classname>RemoteStore</classname> supports multiple
-    concurrent connections to the daemon. This is primarily useful in
-    multi-threaded programs such as
-    <command>hydra-queue-runner</command>.</para>
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>This release has contributions from
-
-Adrien Devresse,
-Alexander Ried,
-Alex Cruice,
-Alexey Shmalko,
-AmineChikhaoui,
-Andy Wingo,
-Aneesh Agrawal,
-Anthony Cowley,
-Armijn Hemel,
-aszlig,
-Ben Gamari,
-Benjamin Hipple,
-Benjamin Staffin,
-Benno Fünfstück,
-Bjørn Forsman,
-Brian McKenna,
-Charles Strahan,
-Chase Adams,
-Chris Martin,
-Christian Theune,
-Chris Warburton,
-Daiderd Jordan,
-Dan Connolly,
-Daniel Peebles,
-Dan Peebles,
-davidak,
-David McFarland,
-Dmitry Kalinkin,
-Domen Kožar,
-Eelco Dolstra,
-Emery Hemingway,
-Eric Litak,
-Eric Wolf,
-Fabian Schmitthenner,
-Frederik Rietdijk,
-Gabriel Gonzalez,
-Giorgio Gallo,
-Graham Christensen,
-Guillaume Maudoux,
-Harmen,
-Iavael,
-James Broadhead,
-James Earl Douglas,
-Janus Troelsen,
-Jeremy Shaw,
-Joachim Schiele,
-Joe Hermaszewski,
-Joel Moberg,
-Johannes 'fish' Ziemke,
-Jörg Thalheim,
-Jude Taylor,
-kballou,
-Keshav Kini,
-Kjetil Orbekk,
-Langston Barrett,
-Linus Heckemann,
-Ludovic Courtès,
-Manav Rathi,
-Marc Scholten,
-Markus Hauck,
-Matt Audesse,
-Matthew Bauer,
-Matthias Beyer,
-Matthieu Coudron,
-N1X,
-Nathan Zadoks,
-Neil Mayhew,
-Nicolas B. Pierron,
-Niklas Hambüchen,
-Nikolay Amiantov,
-Ole Jørgen Brønner,
-Orivej Desh,
-Peter Simons,
-Peter Stuart,
-Pyry Jahkola,
-regnat,
-Renzo Carbonara,
-Rhys,
-Robert Vollmert,
-Scott Olson,
-Scott R. Parish,
-Sergei Trofimovich,
-Shea Levy,
-Sheena Artrip,
-Spencer Baugh,
-Stefan Junker,
-Susan Potter,
-Thomas Tuegel,
-Timothy Allen,
-Tristan Hume,
-Tuomas Tynkkynen,
-tv,
-Tyson Whitehead,
-Vladimír Čunát,
-Will Dietz,
-wmertens,
-Wout Mertens,
-zimbatm and
-Zoran Plesivčak.
-</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
deleted file mode 100644
index 16c243fc1..000000000
--- a/doc/manual/release-notes/rl-2.1.xml
+++ /dev/null
@@ -1,133 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-2.1">
-
-<title>Release 2.1 (2018-09-02)</title>
-
-<para>This is primarily a bug fix release. It also reduces memory
-consumption in certain situations. In addition, it has the following
-new features:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>The Nix installer will no longer default to the Multi-User
-    installation for macOS. You can still <link
-    linkend="sect-multi-user-installation">instruct the installer to
-    run in multi-user mode</link>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The Nix installer now supports performing a Multi-User
-    installation for Linux computers which are running systemd. You
-    can <link
-    linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the
-    <option>--daemon</option> flag to the installer: <command>sh &lt;(curl
-    https://nixos.org/nix/install) --daemon</command>.
-    </para>
-
-    <para>The multi-user installer cannot handle systems with SELinux.
-    If your system has SELinux enabled, you can <link
-    linkend="sect-single-user-installation">force the installer to run
-    in single-user mode</link>.</para>
-  </listitem>
-
-  <listitem>
-    <para>New builtin functions:
-    <literal>builtins.bitAnd</literal>,
-    <literal>builtins.bitOr</literal>,
-    <literal>builtins.bitXor</literal>,
-    <literal>builtins.fromTOML</literal>,
-    <literal>builtins.concatMap</literal>,
-    <literal>builtins.mapAttrs</literal>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The S3 binary cache store now supports uploading NARs larger
-    than 5 GiB.</para>
-  </listitem>
-
-  <listitem>
-    <para>The S3 binary cache store now supports uploading to
-    S3-compatible services with the <literal>endpoint</literal>
-    option.</para>
-  </listitem>
-
-  <listitem>
-    <para>The flag <option>--fallback</option> is no longer required
-    to recover from disappeared NARs in binary caches.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix-daemon</command> now respects
-    <option>--store</option>.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix run</command> now respects
-    <varname>nix-support/propagated-user-env-packages</varname>.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>This release has contributions from
-
-Adrien Devresse,
-Aleksandr Pashkov,
-Alexandre Esteves,
-Amine Chikhaoui,
-Andrew Dunham,
-Asad Saeeduddin,
-aszlig,
-Ben Challenor,
-Ben Gamari,
-Benjamin Hipple,
-Bogdan Seniuc,
-Corey O'Connor,
-Daiderd Jordan,
-Daniel Peebles,
-Daniel Poelzleithner,
-Danylo Hlynskyi,
-Dmitry Kalinkin,
-Domen Kožar,
-Doug Beardsley,
-Eelco Dolstra,
-Erik Arvstedt,
-Félix Baylac-Jacqué,
-Gleb Peregud,
-Graham Christensen,
-Guillaume Maudoux,
-Ivan Kozik,
-John Arnold,
-Justin Humm,
-Linus Heckemann,
-Lorenzo Manacorda,
-Matthew Justin Bauer,
-Matthew O'Gorman,
-Maximilian Bosch,
-Michael Bishop,
-Michael Fiano,
-Michael Mercier,
-Michael Raskin,
-Michael Weiss,
-Nicolas Dudebout,
-Peter Simons,
-Ryan Trinkle,
-Samuel Dionne-Riel,
-Sean Seefried,
-Shea Levy,
-Symphorien Gibol,
-Tim Engler,
-Tim Sears,
-Tuomas Tynkkynen,
-volth,
-Will Dietz,
-Yorick van Pelt and
-zimbatm.
-</para>
-
-</section>
diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml
deleted file mode 100644
index cc992fabb..000000000
--- a/doc/manual/release-notes/rl-2.2.xml
+++ /dev/null
@@ -1,143 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-2.2">
-
-<title>Release 2.2 (2019-01-11)</title>
-
-<para>This is primarily a bug fix release. It also has the following
-changes:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>In derivations that use structured attributes (i.e. that
-    specify set the <varname>__structuredAttrs</varname> attribute to
-    <literal>true</literal> to cause all attributes to be passed to
-    the builder in JSON format), you can now specify closure checks
-    per output, e.g.:
-
-<programlisting>
-outputChecks."out" = {
-  # The closure of 'out' must not be larger than 256 MiB.
-  maxClosureSize = 256 * 1024 * 1024;
-
-  # It must not refer to C compiler or to the 'dev' output.
-  disallowedRequisites = [ stdenv.cc "dev" ];
-};
-
-outputChecks."dev" = {
-  # The 'dev' output must not be larger than 128 KiB.
-  maxSize = 128 * 1024;
-};
-</programlisting>
-
-    </para>
-  </listitem>
-
-
-  <listitem>
-    <para>The derivation attribute
-    <varname>requiredSystemFeatures</varname> is now enforced for
-    local builds, and not just to route builds to remote builders.
-    The supported features of a machine can be specified through the
-    configuration setting <varname>system-features</varname>.</para>
-
-    <para>By default, <varname>system-features</varname> includes
-    <literal>kvm</literal> if <filename>/dev/kvm</filename>
-    exists. For compatibility, it also includes the pseudo-features
-    <literal>nixos-test</literal>, <literal>benchmark</literal> and
-    <literal>big-parallel</literal> which are used by Nixpkgs to route
-    builds to particular Hydra build machines.</para>
-
-  </listitem>
-
-  <listitem>
-    <para>Sandbox builds are now enabled by default on Linux.</para>
-  </listitem>
-
-  <listitem>
-    <para>The new command <command>nix doctor</command> shows
-    potential issues with your Nix installation.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <literal>fetchGit</literal> builtin function now uses a
-    caching scheme that puts different remote repositories in distinct
-    local repositories, rather than a single shared repository. This
-    may require more disk space but is faster.</para>
-  </listitem>
-
-  <listitem>
-    <para>The <literal>dirOf</literal> builtin function now works on
-    relative paths.</para>
-  </listitem>
-
-  <listitem>
-    <para>Nix now supports <link
-    xlink:href="https://www.w3.org/TR/SRI/">SRI hashes</link>,
-    allowing the hash algorithm and hash to be specified in a single
-    string. For example, you can write:
-
-<programlisting>
-import &lt;nix/fetchurl.nix> {
-  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
-  hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ=";
-};
-</programlisting>
-
-    instead of
-
-<programlisting>
-import &lt;nix/fetchurl.nix> {
-  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
-  sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4";
-};
-</programlisting>
-
-    </para>
-
-    <para>In fixed-output derivations, the
-    <varname>outputHashAlgo</varname> attribute is no longer mandatory
-    if <varname>outputHash</varname> specifies the hash.</para>
-
-    <para><command>nix hash-file</command> and <command>nix
-    hash-path</command> now print hashes in SRI format by
-    default. They also use SHA-256 by default instead of SHA-512
-    because that's what we use most of the time in Nixpkgs.</para>
-  </listitem>
-
-  <listitem>
-    <para>Integers are now 64 bits on all platforms.</para>
-  </listitem>
-
-  <listitem>
-    <para>The evaluator now prints profiling statistics (enabled via
-    the <literal>NIX_SHOW_STATS</literal> and
-    <literal>NIX_COUNT_CALLS</literal> environment variables) in JSON
-    format.</para>
-  </listitem>
-
-  <listitem>
-    <para>The option <option>--xml</option> in <command>nix-store
-    --query</command> has been removed. Instead, there now is an
-    option <option>--graphml</option> to output the dependency graph
-    in GraphML format.</para>
-  </listitem>
-
-  <listitem>
-    <para>All <filename>nix-*</filename> commands are now symlinks to
-    <filename>nix</filename>. This saves a bit of disk space.</para>
-  </listitem>
-
-  <listitem>
-    <para><command>nix repl</command> now uses
-    <literal>libeditline</literal> or
-    <literal>libreadline</literal>.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
-
diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
deleted file mode 100644
index 83accf33e..000000000
--- a/doc/manual/release-notes/rl-2.3.xml
+++ /dev/null
@@ -1,91 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-2.3">
-
-<title>Release 2.3 (2019-09-04)</title>
-
-<para>This is primarily a bug fix release. However, it makes some
-incompatible changes:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Nix now uses BSD file locks instead of POSIX file
-    locks. Because of this, you should not use Nix 2.3 and previous
-    releases at the same time on a Nix store.</para>
-  </listitem>
-
-</itemizedlist>
-
-<para>It also has the following changes:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para><function>builtins.fetchGit</function>'s <varname>ref</varname>
-    argument now allows specifying an absolute remote ref.
-    Nix will automatically prefix <varname>ref</varname> with
-    <literal>refs/heads</literal> only if <varname>ref</varname> doesn't
-    already begin with <literal>refs/</literal>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The installer now enables sandboxing by default on Linux when the
-    system has the necessary kernel support.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The <literal>max-jobs</literal> setting now defaults to 1.</para>
-  </listitem>
-
-  <listitem>
-    <para>New builtin functions:
-    <literal>builtins.isPath</literal>,
-    <literal>builtins.hashFile</literal>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The <command>nix</command> command has a new
-    <option>--print-build-logs</option> (<option>-L</option>) flag to
-    print build log output to stderr, rather than showing the last log
-    line in the progress bar. To distinguish between concurrent
-    builds, log lines are prefixed by the name of the package.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>Builds are now executed in a pseudo-terminal, and the
-    <literal>TERM</literal> environment variable is set to
-    <literal>xterm-256color</literal>. This allows many programs
-    (e.g. <command>gcc</command>, <command>clang</command>,
-    <command>cmake</command>) to print colorized log output.</para>
-  </listitem>
-
-  <listitem>
-    <para>Add <option>--no-net</option> convenience flag. This flag
-    disables substituters; sets the <literal>tarball-ttl</literal>
-    setting to infinity (ensuring that any previously downloaded files
-    are considered current); and disables retrying downloads and sets
-    the connection timeout to the minimum. This flag is enabled
-    automatically if there are no configured non-loopback network
-    interfaces.</para>
-  </listitem>
-
-  <listitem>
-    <para>Add a <literal>post-build-hook</literal> setting to run a
-    program after a build has succeeded.</para>
-  </listitem>
-
-  <listitem>
-    <para>Add a <literal>trace-function-calls</literal> setting to log
-    the duration of Nix function calls to stderr.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>
diff --git a/doc/manual/schemas.xml b/doc/manual/schemas.xml
deleted file mode 100644
index 691a517b9..000000000
--- a/doc/manual/schemas.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-<?xml version="1.0"?>
-<locatingRules xmlns="http://thaiopensource.com/ns/locating-rules/1.0">
-  <uri pattern="*.xml" typeId="DocBook"/>
-</locatingRules>

From 2c7557481b4c9d5113a65cc7a75c8acc18031f4e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 24 Jul 2020 21:02:51 +0000
Subject: [PATCH 046/882] `queryDerivationOutputMap` no longer assumes all
 outputs have a mapping

This assumption is broken by CA derivations. Making a PR now to do the
breaking daemon change as soon as possible (if it is already too late,
we can bump protocol intead).
---
 src/libstore/build.cc           | 10 +++++--
 src/libstore/daemon.cc          |  4 +--
 src/libstore/local-store.cc     | 12 +++++---
 src/libstore/local-store.hh     |  2 +-
 src/libstore/remote-store.cc    | 31 +++++++-------------
 src/libstore/remote-store.hh    |  2 +-
 src/libstore/store-api.cc       | 13 ++++++++-
 src/libstore/store-api.hh       | 10 +++++--
 src/libstore/worker-protocol.hh | 52 ++++++++++++++++++++++++++++++++-
 src/nix-env/nix-env.cc          |  2 +-
 10 files changed, 102 insertions(+), 36 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 62294a08c..3a55f24cb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2756,8 +2756,12 @@ struct RestrictedStore : public LocalFSStore
     void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override
-    { throw Error("queryDerivationOutputMap"); }
+    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override
+    {
+        if (!goal.isAllowed(path))
+            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
+        return next->queryDerivationOutputMap(path);
+    }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }
@@ -4918,7 +4922,7 @@ void Worker::waitForInput()
         std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
             if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = read(k, buffer.data(), buffer.size());
+                ssize_t rd = ::read(k, buffer.data(), buffer.size());
                 // FIXME: is there a cleaner way to handle pt close
                 // than EIO? Is this even standard?
                 if (rd == 0 || (rd == -1 && errno == EIO)) {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 7e16529a5..82833f5c3 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -325,9 +325,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputMap: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        OutputPathMap outputs = store->queryDerivationOutputMap(path);
+        auto outputs = store->queryDerivationOutputMap(path);
         logger->stopWork();
-        writeOutputPathMap(*store, to, outputs);
+        write(*store, to, outputs);
         break;
     }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 9ea71170f..4db4f7d6a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -774,17 +774,21 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> LocalStore::queryDerivationOutputMap(const StorePath & path)
 {
-    return retrySQLite<OutputPathMap>([&]() {
+    std::map<std::string, std::optional<StorePath>> outputs;
+    BasicDerivation drv = readDerivation(path);
+    for (auto & [outName, _] : drv.outputs) {
+        outputs.insert_or_assign(outName, std::nullopt);
+    }
+    return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
         auto state(_state.lock());
 
         auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
             (queryValidPathId(*state, path)));
 
-        OutputPathMap outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.emplace(
+            outputs.insert_or_assign(
                 useQueryDerivationOutputs.getStr(0),
                 parseStorePath(useQueryDerivationOutputs.getStr(1))
             );
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 355c2814f..56ef1d251 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -133,7 +133,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 9af4364b7..3b3cdbc02 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -39,30 +39,21 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
         out << store.printStorePath(i);
 }
 
-std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
+
+StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
 {
-    std::map<string, StorePath> pathMap;
-    auto rawInput = readStrings<Strings>(from);
-    if (rawInput.size() % 2)
-        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
-    auto curInput = rawInput.begin();
-    while (curInput != rawInput.end()) {
-        auto thisKey = *curInput++;
-        auto thisValue = *curInput++;
-        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
-    }
-    return pathMap;
+	auto path = readString(from);
+	return store.parseStorePath(path);
 }
 
-void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
+
+void write(const Store & store, Sink & out, const StorePath & storePath)
 {
-    out << 2*pathMap.size();
-    for (auto & i : pathMap) {
-        out << i.first;
-        out << store.printStorePath(i.second);
-    }
+	auto path = store.printStorePath(storePath);
+	out << path;
 }
 
+
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : Store(params)
@@ -445,12 +436,12 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> RemoteStore::queryDerivationOutputMap(const StorePath & path)
 {
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputMap << printStorePath(path);
     conn.processStderr();
-    return readOutputPathMap(*this, conn->from);
+    return read(*this, conn->from, Proxy<std::map<std::string, std::optional<StorePath>>> {});
 
 }
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 3c1b78b6a..b39a8de48 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -51,7 +51,7 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ab21b0bd5..a624fafdd 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -330,9 +330,20 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
+OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path) {
+	auto resp = queryDerivationOutputMap(path);
+	OutputPathMap result;
+	for (auto & [outName, optOutPath] : resp) {
+		if (!optOutPath)
+			throw Error("output '%s' has no store path mapped to it", outName);
+		result.insert_or_assign(outName, *optOutPath);
+	}
+	return result;
+}
+
 StorePathSet Store::queryDerivationOutputs(const StorePath & path)
 {
-    auto outputMap = this->queryDerivationOutputMap(path);
+    auto outputMap = this->queryDerivationOutputMapAssumeTotal(path);
     StorePathSet outputPaths;
     for (auto & i: outputMap) {
         outputPaths.emplace(std::move(i.second));
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d1cb2035f..a83a2ff10 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -423,10 +423,16 @@ public:
     /* Query the outputs of the derivation denoted by `path'. */
     virtual StorePathSet queryDerivationOutputs(const StorePath & path);
 
-    /* Query the mapping outputName=>outputPath for the given derivation */
-    virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
+    /* Query the mapping outputName => outputPath for the given derivation. All
+       outputs are mentioned so ones mising the mapping are mapped to
+       `std::nullopt`.  */
+    virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path)
     { unsupported("queryDerivationOutputMap"); }
 
+    /* Query the mapping outputName=>outputPath for the given derivation.
+       Assume every output has a mapping and throw an exception otherwise. */
+    OutputPathMap queryDerivationOutputMapAssumeTotal(const StorePath & path);
+
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
     virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 8b538f6da..6a6e29640 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -70,6 +70,56 @@ template<class T> T readStorePaths(const Store & store, Source & from);
 
 void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
 
-void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
+/* To guide overloading */
+template<typename T>
+struct Proxy {};
+
+template<typename T>
+std::map<std::string, T> read(const Store & store, Source & from, Proxy<std::map<std::string, T>> _)
+{
+    std::map<string, T> resMap;
+    auto size = (size_t)readInt(from);
+    while (size--) {
+        auto thisKey = readString(from);
+        resMap.insert_or_assign(std::move(thisKey), read(store, from, Proxy<T> {}));
+    }
+    return resMap;
+}
+
+template<typename T>
+void write(const Store & store, Sink & out, const std::map<string, T> & resMap)
+{
+    out << resMap.size();
+    for (auto & i : resMap) {
+        out << i.first;
+        write(store, out, i.second);
+    }
+}
+
+template<typename T>
+std::optional<T> read(const Store & store, Source & from, Proxy<std::optional<T>> _)
+{
+    auto tag = readNum<uint8_t>(from);
+    switch (tag) {
+    case 0:
+        return std::nullopt;
+    case 1:
+        return read(store, from, Proxy<T> {});
+    default:
+        throw Error("got an invalid tag bit for std::optional: %#04x", tag);
+    }
+}
+
+template<typename T>
+void write(const Store & store, Sink & out, const std::optional<T> & optVal)
+{
+    out << (optVal ? 1 : 0);
+    if (optVal)
+        write(store, out, *optVal);
+}
+
+StorePath read(const Store & store, Source & from, Proxy<StorePath> _);
+
+void write(const Store & store, Sink & out, const StorePath & storePath);
 
 }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index ddd036070..d36804658 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -381,7 +381,7 @@ static void queryInstSources(EvalState & state,
 
                 if (path.isDerivation()) {
                     elem.setDrvPath(state.store->printStorePath(path));
-                    auto outputs = state.store->queryDerivationOutputMap(path);
+                    auto outputs = state.store->queryDerivationOutputMapAssumeTotal(path);
                     elem.setOutPath(state.store->printStorePath(outputs.at("out")));
                     if (name.size() >= drvExtension.size() &&
                         string(name, name.size() - drvExtension.size()) == drvExtension)

From 6ccfdb79c7779f9eedeea73a21df694551c9893e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 21 Jul 2020 23:38:18 +0200
Subject: [PATCH 047/882] libutil/logging: extend `internal-json` logger to
 make it more machine-readable

The new error-format is pretty nice from a UX point-of-view, however
it's fairly hard to parse the output e.g. for editor plugins such as
vim-ale[1] that use `nix-instantiate --parse` to determine syntax errors in
Nix expression files.

This patch extends the `internal-json` logger by adding the fields
`line`, `column` and `file` to easily locate an error in a file and the
field `raw_msg` which contains the error-message itself without
code-lines and additional helpers.

An exemplary output may look like this:

```
[nix-shell]$ ./inst/bin/nix-instantiate ~/test.nix --log-format minimal
{"action":"msg","column":1,"file":"/home/ma27/test.nix","level":0,"line":4,"raw_msg":"syntax error, unexpected IF, expecting $end","msg":"<full error-msg with code-lines etc>"}
```

[1] https://github.com/dense-analysis/ale
---
 doc/manual/command-ref/opt-common.xml |  9 ++++++++-
 src/libutil/logging.cc                | 27 +++++++++++++++++++++++++++
 src/libutil/tests/logging.cc          | 18 ++++++++++++++++++
 3 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index a68eef1d0..093bc2526 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -106,7 +106,14 @@
     </varlistentry>
 
     <varlistentry><term>internal-json</term>
-    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
+     <listitem>
+      <para>Outputs the logs in a structured manner.</para>
+      <warning>
+       <para>
+        While the schema itself is relatively stable, the format of the error-messages (namely of the <literal>msg</literal>-field) can change between several releases.
+       </para>
+      </warning>
+     </listitem>
     </varlistentry>
 
     <varlistentry><term>bar</term>
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 832aee783..cbbf64395 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -184,6 +184,33 @@ struct JSONLogger : Logger {
         json["action"] = "msg";
         json["level"] = ei.level;
         json["msg"] = oss.str();
+        json["raw_msg"] = ei.hint->str();
+
+        if (ei.errPos.has_value() && (*ei.errPos)) {
+            json["line"] = ei.errPos->line;
+            json["column"] = ei.errPos->column;
+            json["file"] = ei.errPos->file;
+        } else {
+            json["line"] = nullptr;
+            json["column"] = nullptr;
+            json["file"] = nullptr;
+        }
+
+        if (loggerSettings.showTrace.get() && !ei.traces.empty()) {
+            nlohmann::json traces = nlohmann::json::array();
+            for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) {
+                nlohmann::json stackFrame;
+                stackFrame["raw_msg"] = iter->hint.str();
+                if (iter->pos.has_value() && (*iter->pos)) {
+                    stackFrame["line"] = iter->pos->line;
+                    stackFrame["column"] = iter->pos->column;
+                    stackFrame["file"] = iter->pos->file;
+                }
+                traces.push_back(stackFrame);
+            }
+
+            json["trace"] = traces;
+        }
 
         write(json);
     }
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index ad588055f..7e53f17c6 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -34,6 +34,24 @@ namespace nix {
         }
     }
 
+    TEST(logEI, jsonOutput) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("random.nix");
+        testing::internal::CaptureStderr();
+
+        makeJSONLogger(*logger)->logEI({
+                .name = "error name",
+                .description = "error without any code lines.",
+                .hint = hintfmt("this hint has %1% templated %2%!!",
+                    "yellow",
+                    "values"),
+                .errPos = Pos(foFile, problem_file, 02, 13)
+            });
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
+    }
+
     TEST(logEI, appendingHintsToPreviousError) {
 
         MakeError(TestError, Error);

From d564ac1c5086bda75388d21a25583b2c9809f086 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Mon, 27 Jul 2020 16:42:02 -0400
Subject: [PATCH 048/882] Offer a safer interface for pathOpt

The new interface we offer provides a way of getting all the
DerivationOutputs with the storePaths directly, based on the observation
that it's the most common usecase.
---
 src/libstore/derivations.cc | 21 +++++++++++++++++++++
 src/libstore/derivations.hh | 19 +++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index c88bb3c6d..944d89fe5 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -581,6 +581,27 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
+DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
+    DerivationOutputsAndPaths outsAndPaths;
+    for (auto output : outputs)
+        outsAndPaths.insert(std::make_pair(
+            output.first,
+            std::make_pair(output.second, output.second.path(store, name))
+            )
+        );
+    return outsAndPaths;
+}
+
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+    DerivationOutputsAndOptPaths outsAndOptPaths;
+    for (auto output : outputs)
+        outsAndOptPaths.insert(std::make_pair(
+            output.first,
+            std::make_pair(output.second, output.second.pathOpt(store, output.first))
+            )
+        );
+    return outsAndOptPaths;
+}
 
 std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
     auto nameWithSuffix = drvPath.name();
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index b1cda85cb..b0cb870a2 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -41,6 +41,9 @@ struct DerivationOutput
         DerivationOutputFloating
     > output;
     std::optional<HashType> hashAlgoOpt(const Store & store) const;
+    /* Note, when you use this function you should make sure that you're passing
+       the right derivation name. When in doubt, you should use the safer
+       interface provided by BasicDerivation::outputsAndPaths */
     std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
     /* DEPRECATED: Remove after CA drvs are fully implemented */
     StorePath path(const Store & store, std::string_view drvName) const {
@@ -52,6 +55,15 @@ struct DerivationOutput
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
 
+/* These are analogues to the previous DerivationOutputs data type, but they
+   also contains, for each output, the (optional) store path in which it would
+   be written. To calculate values of these types, see the corresponding
+   functions in BasicDerivation */
+typedef std::map<string, std::pair<DerivationOutput, StorePath>>
+  DerivationOutputsAndPaths;
+typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
+  DerivationOutputsAndOptPaths;
+
 /* For inputs that are sub-derivations, we specify exactly which
    output IDs we are interested in. */
 typedef std::map<StorePath, StringSet> DerivationInputs;
@@ -101,6 +113,13 @@ struct BasicDerivation
     /* Return the output names of a derivation. */
     StringSet outputNames() const;
 
+    /* Calculates the maps that contains all the DerivationOutputs, but
+       augmented with knowledge of the Store paths they would be written into.
+       The first one of these functions will be removed when the CA work is
+       completed */
+    DerivationOutputsAndPaths outputsAndPaths(const Store & store) const;
+    DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
+
     static std::string_view nameFromPath(const StorePath & storePath);
 };
 

From 7ef1e3cd1429f336c568ccf4d59cd89cde68efbd Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Tue, 28 Jul 2020 13:59:24 -0400
Subject: [PATCH 049/882] Use the new interface

---
 perl/lib/Nix/Store.xs       |  4 +-
 src/libexpr/primops.cc      |  8 ++--
 src/libstore/build.cc       | 84 ++++++++++++++++++-------------------
 src/libstore/derivations.cc | 16 +++----
 src/libstore/local-store.cc |  4 +-
 src/libstore/misc.cc        |  6 +--
 src/nix-store/nix-store.cc  |  8 ++--
 src/nix/installables.cc     |  4 +-
 src/nix/repl.cc             |  4 +-
 src/nix/show-derivation.cc  |  6 +--
 10 files changed, 72 insertions(+), 72 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index ec85b844a..7c3416f0b 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -303,10 +303,10 @@ SV * derivationFromPath(char * drvPath)
             hash = newHV();
 
             HV * outputs = newHV();
-            for (auto & i : drv.outputs)
+            for (auto & i : drv.outputsAndPaths(*store()))
                 hv_store(
                     outputs, i.first.c_str(), i.first.size(),
-                    newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
+                    newSVpv(store()->printStorePath(i.second.second).c_str(), 0),
                     0);
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d12d571ad..ee0944fba 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -113,9 +113,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         state.mkList(*outputsVal, drv.outputs.size());
         unsigned int outputs_index = 0;
 
-        for (const auto & o : drv.outputs) {
+        for (const auto & o : drv.outputsAndPaths(*state.store)) {
             v2 = state.allocAttr(w, state.symbols.create(o.first));
-            mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
+            mkString(*v2, state.store->printStorePath(o.second.second), {"!" + o.first + "!" + path});
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
@@ -830,9 +830,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     state.mkAttrs(v, 1 + drv.outputs.size());
     mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
-    for (auto & i : drv.outputs) {
+    for (auto & i : drv.outputsAndPaths(*state.store)) {
         mkString(*state.allocAttr(v, state.symbols.create(i.first)),
-            state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
+            state.store->printStorePath(i.second.second), {"!" + i.first + "!" + drvPathS});
     }
     v.attrs->sort();
 }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 413fc9c4a..7be60396c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1181,8 +1181,8 @@ void DerivationGoal::haveDerivation()
 
     retrySubstitution = false;
 
-    for (auto & i : drv->outputs)
-        worker.store.addTempRoot(i.second.path(worker.store, drv->name));
+    for (auto & i : drv->outputsAndPaths(worker.store))
+        worker.store.addTempRoot(i.second.second);
 
     /* Check what outputs paths are not already valid. */
     auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@@ -1288,14 +1288,14 @@ void DerivationGoal::repairClosure()
 
     /* Get the output closure. */
     StorePathSet outputClosure;
-    for (auto & i : drv->outputs) {
+    for (auto & i : drv->outputsAndPaths(worker.store)) {
         if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
+        worker.store.computeFSClosure(i.second.second, outputClosure);
     }
 
     /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : drv->outputs)
-        outputClosure.erase(i.second.path(worker.store, drv->name));
+    for (auto & i : drv->outputsAndPaths(worker.store))
+        outputClosure.erase(i.second.second);
 
     /* Get all dependencies of this derivation so that we know which
        derivation is responsible for which path in the output
@@ -1306,8 +1306,8 @@ void DerivationGoal::repairClosure()
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
             Derivation drv = worker.store.derivationFromPath(i);
-            for (auto & j : drv.outputs)
-                outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
+            for (auto & j : drv.outputsAndPaths(worker.store))
+                outputsToDrv.insert_or_assign(j.second.second, i);
         }
 
     /* Check each path (slow!). */
@@ -1466,10 +1466,10 @@ void DerivationGoal::tryToBuild()
 
     /* If any of the outputs already exist but are not valid, delete
        them. */
-    for (auto & i : drv->outputs) {
-        if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
-        debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
-        deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
+    for (auto & i : drv->outputsAndPaths(worker.store)) {
+        if (worker.store.isValidPath(i.second.second)) continue;
+        debug("removing invalid path '%s'", worker.store.printStorePath(i.second.second));
+        deletePath(worker.store.Store::toRealPath(i.second.second));
     }
 
     /* Don't do a remote build if the derivation has the attribute
@@ -1919,8 +1919,8 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
     for (auto & j : paths2) {
         if (j.isDerivation()) {
             Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputs)
-                worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
+            for (auto & k : drv.outputsAndPaths(worker.store))
+                worker.store.computeFSClosure(k.second.second, paths);
         }
     }
 
@@ -2014,8 +2014,8 @@ void DerivationGoal::startBuilder()
     chownToBuilder(tmpDir);
 
     /* Substitute output placeholders with the actual output paths. */
-    for (auto & output : drv->outputs)
-        inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
+    for (auto & output : drv->outputsAndPaths(worker.store))
+        inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.second);
 
     /* Construct the environment passed to the builder. */
     initEnv();
@@ -2199,8 +2199,8 @@ void DerivationGoal::startBuilder()
            rebuilding a path that is in settings.dirsInChroot
            (typically the dependencies of /bin/sh).  Throw them
            out. */
-        for (auto & i : drv->outputs)
-            dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
+        for (auto & i : drv->outputsAndPaths(worker.store))
+            dirsInChroot.erase(worker.store.printStorePath(i.second.second));
 
 #elif __APPLE__
         /* We don't really have any parent prep work to do (yet?)
@@ -2612,8 +2612,8 @@ void DerivationGoal::writeStructuredAttrs()
 
     /* Add an "outputs" object containing the output paths. */
     nlohmann::json outputs;
-    for (auto & i : drv->outputs)
-        outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
+    for (auto & i : drv->outputsAndPaths(worker.store))
+        outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.second), inputRewrites);
     json["outputs"] = outputs;
 
     /* Handle exportReferencesGraph. */
@@ -2815,9 +2815,9 @@ struct RestrictedStore : public LocalFSStore
                 if (!goal.isAllowed(path.path))
                     throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
                 auto drv = derivationFromPath(path.path);
-                for (auto & output : drv.outputs)
+                for (auto & output : drv.outputsAndPaths(*this))
                     if (wantOutput(output.first, path.outputs))
-                        newPaths.insert(output.second.path(*this, drv.name));
+                        newPaths.insert(output.second.second);
             } else if (!goal.isAllowed(path.path))
                 throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
         }
@@ -3616,8 +3616,8 @@ void DerivationGoal::registerOutputs()
        to do anything here. */
     if (hook) {
         bool allValid = true;
-        for (auto & i : drv->outputs)
-            if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
+        for (auto & i : drv->outputsAndPaths(worker.store))
+            if (!worker.store.isValidPath(i.second.second)) allValid = false;
         if (allValid) return;
     }
 
@@ -3638,23 +3638,23 @@ void DerivationGoal::registerOutputs()
        Nix calls. */
     StorePathSet referenceablePaths;
     for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
+    for (auto & i : drv->outputsAndPaths(worker.store)) referenceablePaths.insert(i.second.second);
     for (auto & p : addedPaths) referenceablePaths.insert(p);
 
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
-    for (auto & i : drv->outputs) {
-        auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
-        if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
+    for (auto & i : drv->outputsAndPaths(worker.store)) {
+        auto path = worker.store.printStorePath(i.second.second);
+        if (!missingPaths.count(i.second.second)) continue;
 
         Path actualPath = path;
         if (needsHashRewrite()) {
-            auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
+            auto r = redirectedOutputs.find(i.second.second);
             if (r != redirectedOutputs.end()) {
                 auto redirected = worker.store.Store::toRealPath(r->second);
                 if (buildMode == bmRepair
-                    && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
+                    && redirectedBadOutputs.count(i.second.second)
                     && pathExists(redirected))
                     replaceValidPath(path, redirected);
                 if (buildMode == bmCheck)
@@ -3721,7 +3721,7 @@ void DerivationGoal::registerOutputs()
            hash). */
         std::optional<ContentAddress> ca;
 
-        if (! std::holds_alternative<DerivationOutputInputAddressed>(i.second.output)) {
+        if (! std::holds_alternative<DerivationOutputInputAddressed>(i.second.first.output)) {
             DerivationOutputFloating outputHash;
             std::visit(overloaded {
                 [&](DerivationOutputInputAddressed doi) {
@@ -3736,7 +3736,7 @@ void DerivationGoal::registerOutputs()
                 [&](DerivationOutputFloating dof) {
                     outputHash = dof;
                 },
-            }, i.second.output);
+            }, i.second.first.output);
 
             if (outputHash.method == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
@@ -3753,12 +3753,12 @@ void DerivationGoal::registerOutputs()
                 ? hashPath(outputHash.hashType, actualPath).first
                 : hashFile(outputHash.hashType, actualPath);
 
-            auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
+            auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.second.name());
 
             // true if either floating CA, or incorrect fixed hash.
             bool needsMove = true;
 
-            if (auto p = std::get_if<DerivationOutputFixed>(& i.second.output)) {
+            if (auto p = std::get_if<DerivationOutputFixed>(& i.second.first.output)) {
               Hash & h = p->hash.hash;
               if (h != h2) {
 
@@ -3919,8 +3919,8 @@ void DerivationGoal::registerOutputs()
 
     /* If this is the first round of several, then move the output out of the way. */
     if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & i : drv->outputs) {
-            auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
+        for (auto & i : drv->outputsAndPaths(worker.store)) {
+            auto path = worker.store.printStorePath(i.second.second);
             Path prev = path + checkSuffix;
             deletePath(prev);
             Path dst = path + checkSuffix;
@@ -3937,8 +3937,8 @@ void DerivationGoal::registerOutputs()
     /* Remove the .check directories if we're done. FIXME: keep them
        if the result was not determistic? */
     if (curRound == nrRounds) {
-        for (auto & i : drv->outputs) {
-            Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
+        for (auto & i : drv->outputsAndPaths(worker.store)) {
+            Path prev = worker.store.printStorePath(i.second.second) + checkSuffix;
             deletePath(prev);
         }
     }
@@ -4236,12 +4236,12 @@ void DerivationGoal::flushLine()
 StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
 {
     StorePathSet result;
-    for (auto & i : drv->outputs) {
+    for (auto & i : drv->outputsAndPaths(worker.store)) {
         if (!wantOutput(i.first, wantedOutputs)) continue;
         bool good =
-            worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
-            (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
-        if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
+            worker.store.isValidPath(i.second.second) &&
+            (!checkHash || worker.pathContentsGood(i.second.second));
+        if (good == returnValid) result.insert(i.second.second);
     }
     return result;
 }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 944d89fe5..b9c913788 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -473,12 +473,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
         throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
     case DerivationType::CAFixed: {
         std::map<std::string, Hash> outputHashes;
-        for (const auto & i : drv.outputs) {
-            auto & dof = std::get<DerivationOutputFixed>(i.second.output);
+        for (const auto & i : drv.outputsAndPaths(store)) {
+            auto & dof = std::get<DerivationOutputFixed>(i.second.first.output);
             auto hash = hashString(htSHA256, "fixed:out:"
                 + dof.hash.printMethodAlgo() + ":"
                 + dof.hash.hash.to_string(Base16, false) + ":"
-                + store.printStorePath(i.second.path(store, drv.name)));
+                + store.printStorePath(i.second.second));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
         return outputHashes;
@@ -532,8 +532,8 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 StorePathSet BasicDerivation::outputPaths(const Store & store) const
 {
     StorePathSet paths;
-    for (auto & i : outputs)
-        paths.insert(i.second.path(store, name));
+    for (auto & i : outputsAndPaths(store))
+        paths.insert(i.second.second);
     return paths;
 }
 
@@ -642,9 +642,9 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
-    for (auto & i : drv.outputs) {
+    for (auto & i : drv.outputsAndPaths(store)) {
         out << i.first
-            << store.printStorePath(i.second.path(store, drv.name));
+            << store.printStorePath(i.second.second);
         std::visit(overloaded {
             [&](DerivationOutputInputAddressed doi) {
                 out << "" << "";
@@ -657,7 +657,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
                 out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
                     << "";
             },
-        }, i.second.output);
+        }, i.second.first.output);
     }
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0ca21bc40..a87dde1c5 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -618,11 +618,11 @@ uint64_t LocalStore::addValidPath(State & state,
            registration above is undone. */
         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 
-        for (auto & i : drv.outputs) {
+        for (auto & i : drv.outputsAndPaths(*this)) {
             state.stmtAddDerivationOutput.use()
                 (id)
                 (i.first)
-                (printStorePath(i.second.path(*this, drv.name)))
+                (printStorePath(i.second.second))
                 .exec();
         }
     }
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index c4d22a634..46553fecb 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -196,10 +196,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             ParsedDerivation parsedDrv(StorePath(path.path), *drv);
 
             PathSet invalid;
-            for (auto & j : drv->outputs)
+            for (auto & j : drv->outputsAndPaths(*this))
                 if (wantOutput(j.first, path.outputs)
-                    && !isValidPath(j.second.path(*this, drv->name)))
-                    invalid.insert(printStorePath(j.second.path(*this, drv->name)));
+                    && !isValidPath(j.second.second))
+                    invalid.insert(printStorePath(j.second.second));
             if (invalid.empty()) return;
 
             if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 9c8874fd4..87888a1d0 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -218,8 +218,8 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
     if (useOutput && storePath.isDerivation()) {
         auto drv = store->derivationFromPath(storePath);
         StorePathSet outputs;
-        for (auto & i : drv.outputs)
-            outputs.insert(i.second.path(*store, drv.name));
+        for (auto & i : drv.outputsAndPaths(*store))
+            outputs.insert(i.second.second);
         return outputs;
     }
     else return {storePath};
@@ -312,8 +312,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 auto i2 = store->followLinksToStorePath(i);
                 if (forceRealise) realisePath({i2});
                 Derivation drv = store->derivationFromPath(i2);
-                for (auto & j : drv.outputs)
-                    cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
+                for (auto & j : drv.outputsAndPaths(*store))
+                    cout << fmt("%1%\n", store->printStorePath(j.second.second));
             }
             break;
         }
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index b245e417e..41c077c47 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -305,8 +305,8 @@ struct InstallableStorePath : Installable
         if (storePath.isDerivation()) {
             std::map<std::string, StorePath> outputs;
             auto drv = store->readDerivation(storePath);
-            for (auto & [name, output] : drv.outputs)
-                outputs.emplace(name, output.path(*store, drv.name));
+            for (auto & i : drv.outputsAndPaths(*store))
+                outputs.emplace(i.first, i.second.second);
             return {
                 Buildable {
                     .drvPath = storePath,
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index fb9050d0d..803d89018 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -485,8 +485,8 @@ bool NixRepl::processLine(string line)
             if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
                 auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-                for (auto & i : drv.outputs)
-                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
+                for (auto & i : drv.outputsAndPaths(*state->store))
+                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.second));
             }
         } else if (command == ":i") {
             runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 25ea19834..87122be87 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -67,9 +67,9 @@ struct CmdShowDerivation : InstallablesCommand
 
             {
                 auto outputsObj(drvObj.object("outputs"));
-                for (auto & output : drv.outputs) {
+                for (auto & output : drv.outputsAndPaths(*store)) {
                     auto outputObj(outputsObj.object(output.first));
-                    outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
+                    outputObj.attr("path", store->printStorePath(output.second.second));
 
                     std::visit(overloaded {
                         [&](DerivationOutputInputAddressed doi) {
@@ -81,7 +81,7 @@ struct CmdShowDerivation : InstallablesCommand
                         [&](DerivationOutputFloating dof) {
                             outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                         },
-                    }, output.second.output);
+                    }, output.second.first.output);
                 }
             }
 

From 13ef7a07b9be1dff894e8156a117ee3248241874 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 30 Jul 2020 15:49:45 -0500
Subject: [PATCH 050/882] Fix build

---
 flake.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flake.nix b/flake.nix
index bf2066157..40d779355 100644
--- a/flake.nix
+++ b/flake.nix
@@ -74,6 +74,7 @@
             # Tests
             buildPackages.git
             buildPackages.mercurial
+            buildPackages.jq
           ];
 
         buildDeps =

From 3537670fefab6c65b4b87837112d64931dcda4cf Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 30 Jul 2020 15:49:52 -0500
Subject: [PATCH 051/882] Only enable static on linux

---
 flake.nix | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/flake.nix b/flake.nix
index 40d779355..20c5089ee 100644
--- a/flake.nix
+++ b/flake.nix
@@ -15,7 +15,8 @@
 
       officialRelease = false;
 
-      systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
+      linuxSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
+      systems = linuxSystems ++ [ "x86_64-darwin" ];
 
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
 
@@ -209,7 +210,7 @@
         # Binary package for various platforms.
         build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix);
 
-        build-static = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix-static);
+        build-static = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static);
 
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
@@ -423,13 +424,14 @@
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
-        build-static = self.hydraJobs.build-static.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
+      } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) {
+        build-static = self.hydraJobs.build-static.${system};
       });
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;
-
+      } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;
         in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {

From 1d0a7b54fa330b041a720932ee4e05dcad1d2d5c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Jul 2020 15:43:25 +0200
Subject: [PATCH 052/882] Enable syntax highlighting

---
 doc/manual/highlight.pack.js                  |   6 +
 doc/manual/local.mk                           |   1 +
 doc/manual/src/advanced-topics/diff-hook.md   | 118 +++--
 .../src/advanced-topics/distributed-builds.md |  50 +-
 .../src/advanced-topics/post-build-hook.md    |  68 +--
 doc/manual/src/command-ref/conf-file.md       |  45 +-
 doc/manual/src/command-ref/env-common.md      |   8 +-
 doc/manual/src/command-ref/nix-build.md       |  38 +-
 doc/manual/src/command-ref/nix-channel.md     |  22 +-
 .../src/command-ref/nix-collect-garbage.md    |   4 +-
 .../src/command-ref/nix-copy-closure.md       |  12 +-
 doc/manual/src/command-ref/nix-env.md         | 292 +++++++-----
 doc/manual/src/command-ref/nix-hash.md        |  50 +-
 doc/manual/src/command-ref/nix-instantiate.md |  92 ++--
 .../src/command-ref/nix-prefetch-url.md       |  22 +-
 doc/manual/src/command-ref/nix-shell.md       | 142 +++---
 doc/manual/src/command-ref/nix-store.md       | 210 ++++----
 doc/manual/src/command-ref/opt-common.md      |  12 +-
 .../src/expressions/advanced-attributes.md    | 122 ++---
 .../src/expressions/arguments-variables.md    |  48 +-
 doc/manual/src/expressions/build-script.md    |  20 +-
 doc/manual/src/expressions/builtins.md        | 448 ++++++++++--------
 doc/manual/src/expressions/derivations.md     |  19 +-
 .../src/expressions/expression-syntax.md      |  66 +--
 doc/manual/src/expressions/generic-builder.md |  34 +-
 .../src/expressions/language-constructs.md    | 240 ++++++----
 doc/manual/src/expressions/language-values.md | 110 +++--
 .../expressions/simple-building-testing.md    |  32 +-
 doc/manual/src/hacking.md                     |  50 +-
 .../src/installation/building-source.md       |  12 +-
 doc/manual/src/installation/env-variables.md  |  20 +-
 .../src/installation/installing-binary.md     |  78 +--
 doc/manual/src/installation/multi-user.md     |  24 +-
 .../src/installation/obtaining-source.md      |   4 +-
 doc/manual/src/introduction.md                |  34 +-
 .../package-management/basic-package-mgmt.md  |  82 ++--
 .../binary-cache-substituter.md               |  16 +-
 doc/manual/src/package-management/channels.md |  16 +-
 .../package-management/garbage-collection.md  |  24 +-
 .../garbage-collector-roots.md                |   4 +-
 doc/manual/src/package-management/profiles.md |  48 +-
 .../src/package-management/s3-substituter.md  |  90 ++--
 .../src/package-management/ssh-substituter.md |  20 +-
 doc/manual/src/quick-start.md                 |  68 ++-
 flake.nix                                     |   1 +
 src/libstore/nar-accessor.cc                  |   3 +-
 46 files changed, 1770 insertions(+), 1155 deletions(-)
 create mode 100644 doc/manual/highlight.pack.js

diff --git a/doc/manual/highlight.pack.js b/doc/manual/highlight.pack.js
new file mode 100644
index 000000000..fba8b4a5a
--- /dev/null
+++ b/doc/manual/highlight.pack.js
@@ -0,0 +1,6 @@
+/*
+  Highlight.js 10.1.2 (edd73d24)
+  License: BSD-3-Clause
+  Copyright (c) 2006-2020, Ivan Sagalaev
+*/
+var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!Object.hasOwnProperty.call(n,r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}class n{constructor(e){void 0===e.data&&(e.data={}),this.data=e.data}ignoreMatch(){this.ignore=!0}}function t(e){return e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#x27;")}function r(e,...n){var t={};for(const n in e)t[n]=e[n];return n.forEach((function(e){for(const n in e)t[n]=e[n]})),t}function a(e){return e.nodeName.toLowerCase()}var i=Object.freeze({__proto__:null,escapeHTML:t,inherit:r,nodeStream:function(e){var n=[];return function e(t,r){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?r+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:r,node:i}),r=e(i,r),a(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:r,node:i}));return r}(e,0),n},mergeStreams:function(e,n,r){var i=0,s="",o=[];function l(){return e.length&&n.length?e[0].offset!==n[0].offset?e[0].offset<n[0].offset?e:n:"start"===n[0].event?e:n:e.length?e:n}function c(e){s+="<"+a(e)+[].map.call(e.attributes,(function(e){return" "+e.nodeName+'="'+t(e.value)+'"'})).join("")+">"}function u(e){s+="</"+a(e)+">"}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||n.length;){var g=l();if(s+=t(r.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+t(r.substr(i))}});const s="</span>",o=e=>!!e.kind;class l{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=t(e)}openNode(e){if(!o(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){o(e)&&(this.buffer+=s)}value(){return this.buffer}span(e){this.buffer+=`<span class="${e}">`}}class c{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){const n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){"string"!=typeof e&&e.children&&(e.children.every(e=>"string"==typeof e)?e.children=[e.children.join("")]:e.children.forEach(e=>{c._collapse(e)}))}}class u extends c{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){const t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new l(this,this.options).value()}finalize(){return!0}}function d(e){return e?"string"==typeof e?e:e.source:null}const g="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",h={begin:"\\\\[\\s\\S]",relevance:0},f={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[h]},p={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[h]},b={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,t={}){var a=r({className:"comment",begin:e,end:n,contains:[]},t);return a.contains.push(b),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):",relevance:0}),a},v=m("//","$"),x=m("/\\*","\\*/"),E=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:g,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",SHEBANG:(e={})=>{const n=/^#![ ]*\//;return e.binary&&(e.begin=function(...e){return e.map(e=>d(e)).join("")}(n,/.*\b/,e.binary,/\b.*/)),r({className:"meta",begin:n,end:/$/,relevance:0,"on:begin":(e,n)=>{0!==e.index&&n.ignoreMatch()}},e)},BACKSLASH_ESCAPE:h,APOS_STRING_MODE:f,QUOTE_STRING_MODE:p,PHRASAL_WORDS_MODE:b,COMMENT:m,C_LINE_COMMENT_MODE:v,C_BLOCK_COMMENT_MODE:x,HASH_COMMENT_MODE:E,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:g,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[h,{begin:/\[/,end:/\]/,relevance:0,contains:[h]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0},END_SAME_AS_BEGIN:function(e){return Object.assign(e,{"on:begin":(e,n)=>{n.data._beginMatch=e[1]},"on:end":(e,n)=>{n.data._beginMatch!==e[1]&&n.ignoreMatch()}})}}),N="of and for in not or if then".split(" ");function w(e,n){return n?+n:function(e){return N.includes(e.toLowerCase())}(e)?0:1}const R=t,y=r,{nodeStream:O,mergeStreams:k}=i,M=Symbol("nomatch");return function(t){var a=[],i=Object.create(null),s=Object.create(null),o=[],l=!0,c=/(^(<[^>]+>|\t|)+|\n)/gm,g="Could not find the language '{}', did you forget to load/include a language module?";const h={disableAutodetect:!0,name:"Plain text",contains:[]};var f={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:null,__emitter:u};function p(e){return f.noHighlightRe.test(e)}function b(e,n,t,r){var a={code:n,language:e};S("before:highlight",a);var i=a.result?a.result:m(a.language,a.code,t,r);return i.code=a.code,S("after:highlight",i),i}function m(e,t,a,s){var o=t;function c(e,n){var t=E.case_insensitive?n[0].toLowerCase():n[0];return Object.prototype.hasOwnProperty.call(e.keywords,t)&&e.keywords[t]}function u(){null!=y.subLanguage?function(){if(""!==A){var e=null;if("string"==typeof y.subLanguage){if(!i[y.subLanguage])return void k.addText(A);e=m(y.subLanguage,A,!0,O[y.subLanguage]),O[y.subLanguage]=e.top}else e=v(A,y.subLanguage.length?y.subLanguage:null);y.relevance>0&&(I+=e.relevance),k.addSublanguage(e.emitter,e.language)}}():function(){if(!y.keywords)return void k.addText(A);let e=0;y.keywordPatternRe.lastIndex=0;let n=y.keywordPatternRe.exec(A),t="";for(;n;){t+=A.substring(e,n.index);const r=c(y,n);if(r){const[e,a]=r;k.addText(t),t="",I+=a,k.addKeyword(n[0],e)}else t+=n[0];e=y.keywordPatternRe.lastIndex,n=y.keywordPatternRe.exec(A)}t+=A.substr(e),k.addText(t)}(),A=""}function h(e){return e.className&&k.openNode(e.className),y=Object.create(e,{parent:{value:y}})}function p(e){return 0===y.matcher.regexIndex?(A+=e[0],1):(L=!0,0)}var b={};function x(t,r){var i=r&&r[0];if(A+=t,null==i)return u(),0;if("begin"===b.type&&"end"===r.type&&b.index===r.index&&""===i){if(A+=o.slice(r.index,r.index+1),!l){const n=Error("0 width match regex");throw n.languageName=e,n.badRule=b.rule,n}return 1}if(b=r,"begin"===r.type)return function(e){var t=e[0],r=e.rule;const a=new n(r),i=[r.__beforeBegin,r["on:begin"]];for(const n of i)if(n&&(n(e,a),a.ignore))return p(t);return r&&r.endSameAsBegin&&(r.endRe=RegExp(t.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),r.skip?A+=t:(r.excludeBegin&&(A+=t),u(),r.returnBegin||r.excludeBegin||(A=t)),h(r),r.returnBegin?0:t.length}(r);if("illegal"===r.type&&!a){const e=Error('Illegal lexeme "'+i+'" for mode "'+(y.className||"<unnamed>")+'"');throw e.mode=y,e}if("end"===r.type){var s=function(e){var t=e[0],r=o.substr(e.index),a=function e(t,r,a){let i=function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(t.endRe,a);if(i){if(t["on:end"]){const e=new n(t);t["on:end"](r,e),e.ignore&&(i=!1)}if(i){for(;t.endsParent&&t.parent;)t=t.parent;return t}}if(t.endsWithParent)return e(t.parent,r,a)}(y,e,r);if(!a)return M;var i=y;i.skip?A+=t:(i.returnEnd||i.excludeEnd||(A+=t),u(),i.excludeEnd&&(A=t));do{y.className&&k.closeNode(),y.skip||y.subLanguage||(I+=y.relevance),y=y.parent}while(y!==a.parent);return a.starts&&(a.endSameAsBegin&&(a.starts.endRe=a.endRe),h(a.starts)),i.returnEnd?0:t.length}(r);if(s!==M)return s}if("illegal"===r.type&&""===i)return 1;if(B>1e5&&B>3*r.index)throw Error("potential infinite loop, way more iterations than matches");return A+=i,i.length}var E=T(e);if(!E)throw console.error(g.replace("{}",e)),Error('Unknown language: "'+e+'"');var _=function(e){function n(n,t){return RegExp(d(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class t{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);const e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n="|"){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i<e.length;i++){var s=r+=1,o=d(e[i]);for(i>0&&(a+=n),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"===l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("===l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;const n=this.matcherRe.exec(e);if(!n)return null;const t=n.findIndex((e,n)=>n>0&&void 0!==e),r=this.matchIndexes[t];return n.splice(0,t),Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];const n=new t;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){const n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;const t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e,n){const t=e.input[e.index-1],r=e.input[e.index+e[0].length];"."!==t&&"."!==r||n.ignoreMatch()}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language.  See documentation.");return function t(s,o){const l=s;if(s.compiled)return l;s.compiled=!0,s.__beforeBegin=null,s.keywords=s.keywords||s.beginKeywords;let c=null;if("object"==typeof s.keywords&&(c=s.keywords.$pattern,delete s.keywords.$pattern),s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,w(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemes&&c)throw Error("ERR: Prefer `keywords.$pattern` to `mode.lexemes`, BOTH are not allowed. (see mode reference) ");return l.keywordPatternRe=n(s.lexemes||c||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__beforeBegin=i),s.begin||(s.begin=/\B|\b/),l.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(l.endRe=n(s.end)),l.terminator_end=d(s.end)||"",s.endsWithParent&&o.terminator_end&&(l.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(l.illegalRe=n(s.illegal)),void 0===s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return r(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?r(e,{starts:e.starts?r(e.starts):null}):Object.isFrozen(e)?r(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){t(e,l)})),s.starts&&t(s.starts,o),l.matcher=function(e){const n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(l),l}(e)}(E),N="",y=s||_,O={},k=new f.__emitter(f);!function(){for(var e=[],n=y;n!==E;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>k.openNode(e))}();var A="",I=0,S=0,B=0,L=!1;try{for(y.matcher.considerAll();;){B++,L?L=!1:(y.matcher.lastIndex=S,y.matcher.considerAll());const e=y.matcher.exec(o);if(!e)break;const n=x(o.substring(S,e.index),e);S=e.index+n}return x(o.substr(S)),k.closeAllNodes(),k.finalize(),N=k.toHTML(),{relevance:I,value:N,language:e,illegal:!1,emitter:k,top:y}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:o.slice(S-100,S+100),mode:n.mode},sofar:N,relevance:0,value:R(o),emitter:k};if(l)return{illegal:!1,relevance:0,value:R(o),emitter:k,language:e,top:y,errorRaised:n};throw n}}function v(e,n){n=n||f.languages||Object.keys(i);var t=function(e){const n={relevance:0,emitter:new f.__emitter(f),value:R(e),illegal:!1,top:h};return n.emitter.addText(e),n}(e),r=t;return n.filter(T).filter(I).forEach((function(n){var a=m(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function x(e){return f.tabReplace||f.useBR?e.replace(c,e=>"\n"===e?f.useBR?"<br>":e:f.tabReplace?e.replace(/\t/g,f.tabReplace):e):e}function E(e){let n=null;const t=function(e){var n=e.className+" ";n+=e.parentNode?e.parentNode.className:"";const t=f.languageDetectRe.exec(n);if(t){var r=T(t[1]);return r||(console.warn(g.replace("{}",t[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?t[1]:"no-highlight"}return n.split(/\s+/).find(e=>p(e)||T(e))}(e);if(p(t))return;S("before:highlightBlock",{block:e,language:t}),f.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(/<br[ /]*>/g,"\n"):n=e;const r=n.textContent,a=t?b(t,r,!0):v(r),i=O(n);if(i.length){const e=document.createElement("div");e.innerHTML=a.value,a.value=k(i,O(e),r)}a.value=x(a.value),S("after:highlightBlock",{block:e,result:a}),e.innerHTML=a.value,e.className=function(e,n,t){var r=n?s[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,t,a.language),e.result={language:a.language,re:a.relevance,relavance:a.relevance},a.second_best&&(e.second_best={language:a.second_best.language,re:a.second_best.relevance,relavance:a.second_best.relevance})}const N=()=>{if(!N.called){N.called=!0;var e=document.querySelectorAll("pre code");a.forEach.call(e,E)}};function T(e){return e=(e||"").toLowerCase(),i[e]||i[s[e]]}function A(e,{languageName:n}){"string"==typeof e&&(e=[e]),e.forEach(e=>{s[e]=n})}function I(e){var n=T(e);return n&&!n.disableAutodetect}function S(e,n){var t=e;o.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(t,{highlight:b,highlightAuto:v,fixMarkup:x,highlightBlock:E,configure:function(e){f=y(f,e)},initHighlighting:N,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",N,!1)},registerLanguage:function(e,n){var r=null;try{r=n(t)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!l)throw n;console.error(n),r=h}r.name||(r.name=e),i[e]=r,r.rawDefinition=n.bind(null,t),r.aliases&&A(r.aliases,{languageName:e})},listLanguages:function(){return Object.keys(i)},getLanguage:T,registerAliases:A,requireLanguage:function(e){var n=T(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:I,inherit:y,addPlugin:function(e){o.push(e)}}),t.debugMode=function(){l=!1},t.safeMode=function(){l=!0},t.versionString="10.1.2";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(t,_),t}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);hljs.registerLanguage("xml",function(){"use strict";return function(e){var n={className:"symbol",begin:"&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;"},a={begin:"\\s",contains:[{className:"meta-keyword",begin:"#?[a-z_][a-z1-9_-]+",illegal:"\\n"}]},s=e.inherit(a,{begin:"\\(",end:"\\)"}),t=e.inherit(e.APOS_STRING_MODE,{className:"meta-string"}),i=e.inherit(e.QUOTE_STRING_MODE,{className:"meta-string"}),c={endsWithParent:!0,illegal:/</,relevance:0,contains:[{className:"attr",begin:"[A-Za-z0-9\\._:-]+",relevance:0},{begin:/=\s*/,relevance:0,contains:[{className:"string",endsParent:!0,variants:[{begin:/"/,end:/"/,contains:[n]},{begin:/'/,end:/'/,contains:[n]},{begin:/[^\s"'=<>`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"<![a-z]",end:">",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"<![a-z]",end:">",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:"<style(?=\\s|>)",end:">",keywords:{name:"style"},contains:[c],starts:{end:"</style>",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:"<script(?=\\s|>)",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"</?",end:"/?>",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}());hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}());hljs.registerLanguage("perl",function(){"use strict";return function(e){var n={$pattern:/[\w.]+/,keyword:"getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qq fileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent shutdown dump chomp connect getsockname die socketpair close flock exists index shmget sub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedir ioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when"},t={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:n},s={begin:"->{",end:"}"},r={variants:[{begin:/\$\d/},{begin:/[\$%@](\^\w\b|#\w+(::\w+)*|{\w+}|\w+(::\w*)*)/},{begin:/[\$%@][^\s\w{]/,relevance:0}]},i=[e.BACKSLASH_ESCAPE,t,r],a=[r,e.HASH_COMMENT_MODE,e.COMMENT("^\\=\\w","\\=cut",{endsWithParent:!0}),s,{className:"string",contains:i,variants:[{begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[",end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*\\<",end:"\\>",relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE]},{begin:"{\\w+}",contains:[],relevance:0},{begin:"-?\\w+\\s*\\=\\>",contains:[],relevance:0}]},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",keywords:"split return print reverse grep",relevance:0,contains:[e.HASH_COMMENT_MODE,{className:"regexp",begin:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",relevance:10},{className:"regexp",begin:"(m|qr)?/",end:"/[a-z]*",contains:[e.BACKSLASH_ESCAPE],relevance:0}]},{className:"function",beginKeywords:"sub",end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE]},{begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$",subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}]}];return t.contains=a,s.contains=a,{name:"Perl",aliases:["pl","pm"],keywords:n,contains:a}}}());hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}());hljs.registerLanguage("ini",function(){"use strict";function e(e){return e?"string"==typeof e?e:e.source:null}function n(...n){return n.map(n=>e(n)).join("")}return function(a){var s={className:"number",relevance:0,variants:[{begin:/([\+\-]+)?[\d]+_[\d_]+/},{begin:a.NUMBER_RE}]},i=a.COMMENT();i.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];var t={className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)}/}]},r={className:"literal",begin:/\bon|off|true|false|yes|no\b/},l={className:"string",contains:[a.BACKSLASH_ESCAPE],variants:[{begin:"'''",end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"'},{begin:"'",end:"'"}]},c={begin:/\[/,end:/\]/,contains:[i,r,t,l,s,"self"],relevance:0},g="("+[/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/].map(n=>e(n)).join("|")+")";return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/,contains:[i,{className:"section",begin:/\[+/,end:/\]+/},{begin:n(g,"(\\s*\\.\\s*",g,")*",n("(?=",/\s*=\s*[^#\s]/,")")),className:"attr",starts:{end:/$/,contains:[i,c,r,t,l,s]}}]}}}());hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}());hljs.registerLanguage("ruby",function(){"use strict";return function(e){var n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",a={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},s={className:"doctag",begin:"@[A-Za-z]+"},i={begin:"#<",end:">"},r=[e.COMMENT("#","$",{contains:[s]}),e.COMMENT("^\\=begin","^\\=end",{contains:[s],relevance:10}),e.COMMENT("^__END__","\\n$")],c={className:"subst",begin:"#\\{",end:"}",keywords:a},t={className:"string",contains:[e.BACKSLASH_ESCAPE,c],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:"%[qQwWx]?\\(",end:"\\)"},{begin:"%[qQwWx]?\\[",end:"\\]"},{begin:"%[qQwWx]?{",end:"}"},{begin:"%[qQwWx]?<",end:">"},{begin:"%[qQwWx]?/",end:"/"},{begin:"%[qQwWx]?%",end:"%"},{begin:"%[qQwWx]?-",end:"-"},{begin:"%[qQwWx]?\\|",end:"\\|"},{begin:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{begin:/<<[-~]?'?(\w+)(?:.|\n)*?\n\s*\1\b/,returnBegin:!0,contains:[{begin:/<<[-~]?'?/},e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,contains:[e.BACKSLASH_ESCAPE,c]})]}]},b={className:"params",begin:"\\(",end:"\\)",endsParent:!0,keywords:a},d=[t,i,{className:"class",beginKeywords:"class module",end:"$|;",illegal:/=/,contains:[e.inherit(e.TITLE_MODE,{begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{begin:"<\\s*",contains:[{begin:"("+e.IDENT_RE+"::)?"+e.IDENT_RE}]}].concat(r)},{className:"function",beginKeywords:"def",end:"$|;",contains:[e.inherit(e.TITLE_MODE,{begin:n}),b].concat(r)},{begin:e.IDENT_RE+"::"},{className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"(\\!|\\?)?:",relevance:0},{className:"symbol",begin:":(?!\\s)",contains:[t,{begin:n}],relevance:0},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{className:"params",begin:/\|/,end:/\|/,keywords:a},{begin:"("+e.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[i,{className:"regexp",contains:[e.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{begin:"/",end:"/[a-z]*"},{begin:"%r{",end:"}[a-z]*"},{begin:"%r\\(",end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}]}].concat(r),relevance:0}].concat(r);c.contains=d,b.contains=d;var g=[{begin:/^\s*=>/,starts:{end:"$",contains:d}},{className:"meta",begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+>|(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>)",starts:{end:"$",contains:d}}];return{name:"Ruby",aliases:["rb","gemspec","podspec","thor","irb"],keywords:a,illegal:/\/\*/,contains:r.concat(g).concat(d)}}}());hljs.registerLanguage("yaml",function(){"use strict";return function(e){var n="true false yes no null",a="[\\w#;/?:@&=+$,.~*\\'()[\\]]+",s={className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable",variants:[{begin:"{{",end:"}}"},{begin:"%{",end:"}"}]}]},i=e.inherit(s,{variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),l={end:",",endsWithParent:!0,excludeEnd:!0,contains:[],keywords:n,relevance:0},t={begin:"{",end:"}",contains:[l],illegal:"\\n",relevance:0},g={begin:"\\[",end:"\\]",contains:[l],illegal:"\\n",relevance:0},b=[{className:"attr",variants:[{begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---s*$",relevance:10},{className:"string",begin:"[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*"},{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:"!\\w+!"+a},{className:"type",begin:"!<"+a+">"},{className:"type",begin:"!"+a},{className:"type",begin:"!!"+a},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta",begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"\\-(?=[ ]|$)",relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{className:"number",begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"},{className:"number",begin:e.C_NUMBER_RE+"\\b"},t,g,s],c=[...b];return c.pop(),c.push(i),l.contains=c,{name:"YAML",case_insensitive:!0,aliases:["yml","YAML"],contains:b}}}());hljs.registerLanguage("haskell",function(){"use strict";return function(e){var n={variants:[e.COMMENT("--","$"),e.COMMENT("{-","-}",{contains:["self"]})]},i={className:"meta",begin:"{-#",end:"#-}"},a={className:"meta",begin:"^#",end:"$"},s={className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},l={begin:"\\(",end:"\\)",illegal:'"',contains:[i,a,{className:"type",begin:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},e.inherit(e.TITLE_MODE,{begin:"[_a-z][\\w']*"}),n]};return{name:"Haskell",aliases:["hs"],keywords:"let in if then else case of where do module import hiding qualified type data newtype deriving class instance as default infix infixl infixr foreign export ccall stdcall cplusplus jvm dotnet safe unsafe family forall mdo proc rec",contains:[{beginKeywords:"module",end:"where",keywords:"module where",contains:[l,n],illegal:"\\W\\.|;"},{begin:"\\bimport\\b",end:"$",keywords:"import qualified as hiding",contains:[l,n],illegal:"\\W\\.|;"},{className:"class",begin:"^(\\s*)?(class|instance)\\b",end:"where",keywords:"class family instance where",contains:[s,l,n]},{className:"class",begin:"\\b(data|(new)?type)\\b",end:"$",keywords:"data family type newtype deriving",contains:[i,s,l,{begin:"{",end:"}",contains:l.contains},n]},{beginKeywords:"default",end:"$",contains:[s,l,n]},{beginKeywords:"infix infixl infixr",end:"$",contains:[e.C_NUMBER_MODE,n]},{begin:"\\bforeign\\b",end:"$",keywords:"foreign import export ccall stdcall cplusplus jvm dotnet safe unsafe",contains:[s,e.QUOTE_STRING_MODE,n]},{className:"meta",begin:"#!\\/usr\\/bin\\/env runhaskell",end:"$"},i,a,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,s,e.inherit(e.TITLE_MODE,{begin:"^[_a-z][\\w']*"}),n,{begin:"->|<-"}]}}}());hljs.registerLanguage("nix",function(){"use strict";return function(e){var n={keyword:"rec with let in inherit assert if else then",literal:"true false or and null",built_in:"import abort baseNameOf dirOf isNull builtins map removeAttrs throw toString derivation"},i={className:"subst",begin:/\$\{/,end:/}/,keywords:n},t={className:"string",contains:[i],variants:[{begin:"''",end:"''"},{begin:'"',end:'"'}]},s=[e.NUMBER_MODE,e.HASH_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,t,{begin:/[a-zA-Z0-9-_]+(\s*=)/,returnBegin:!0,relevance:0,contains:[{className:"attr",begin:/\S+/}]}];return i.contains=s,{name:"Nix",aliases:["nixos"],keywords:n,contains:s}}}());hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const t={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},n={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,t]};t.contains.push(n);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]},i=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10}),c={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b-?[a-z\._]+\b/,keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[i,e.SHEBANG(),c,a,e.HASH_COMMENT_MODE,n,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}());hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}());hljs.registerLanguage("python",function(){"use strict";return function(e){var n={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10",built_in:"Ellipsis NotImplemented",literal:"False None True"},a={className:"meta",begin:/^(>>>|\.\.\.) /},i={className:"subst",begin:/\{/,end:/\}/,keywords:n,illegal:/#/},s={begin:/\{\{/,relevance:0},r={className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/(u|b)?r?'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(u|b)?r?"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(fr|rf|f)'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(fr|rf|f)"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(u|r|ur)'/,end:/'/,relevance:10},{begin:/(u|r|ur)"/,end:/"/,relevance:10},{begin:/(b|br)'/,end:/'/},{begin:/(b|br)"/,end:/"/},{begin:/(fr|rf|f)'/,end:/'/,contains:[e.BACKSLASH_ESCAPE,s,i]},{begin:/(fr|rf|f)"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,i]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},l={className:"number",relevance:0,variants:[{begin:e.BINARY_NUMBER_RE+"[lLjJ]?"},{begin:"\\b(0o[0-7]+)[lLjJ]?"},{begin:e.C_NUMBER_RE+"[lLjJ]?"}]},t={className:"params",variants:[{begin:/\(\s*\)/,skip:!0,className:null},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:["self",a,l,r,e.HASH_COMMENT_MODE]}]};return i.contains=[r,l,a],{name:"Python",aliases:["py","gyp","ipython"],keywords:n,illegal:/(<\/|->|\?)|=>/,contains:[a,l,{beginKeywords:"if",relevance:0},r,e.HASH_COMMENT_MODE,{variants:[{className:"function",beginKeywords:"def"},{className:"class",beginKeywords:"class"}],end:/:/,illegal:/[${=;\n,]/,contains:[e.UNDERSCORE_TITLE_MODE,t,{begin:/->/,endsWithParent:!0,keywords:"None"}]},{className:"meta",begin:/^[\t ]*@/,end:/$/},{begin:/\b(print|exec)\(/}]}}}());
\ No newline at end of file
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 2d730a60e..c9104ad7e 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -29,5 +29,6 @@ install: $(docdir)/manual/index.html
 
 $(docdir)/manual/index.html: $(MANUAL_SRCS)
 	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
+	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
 
 endif
diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md
index e2234147f..7a2622b3d 100644
--- a/doc/manual/src/advanced-topics/diff-hook.md
+++ b/doc/manual/src/advanced-topics/diff-hook.md
@@ -7,17 +7,19 @@ for determining if the results are the same.
 For purposes of demonstration, we'll use the following Nix file,
 `deterministic.nix` for testing:
 
-    let
-      inherit (import <nixpkgs> {}) runCommand;
-    in {
-      stable = runCommand "stable" {} ''
-        touch $out
-      '';
-    
-      unstable = runCommand "unstable" {} ''
-        echo $RANDOM > $out
-      '';
-    }
+```nix
+let
+  inherit (import <nixpkgs> {}) runCommand;
+in {
+  stable = runCommand "stable" {} ''
+    touch $out
+  '';
+
+  unstable = runCommand "unstable" {} ''
+    echo $RANDOM > $out
+  '';
+}
+```
 
 Additionally, `nix.conf` contains:
 
@@ -26,10 +28,12 @@ Additionally, `nix.conf` contains:
 
 where `/etc/nix/my-diff-hook` is an executable file containing:
 
-    #!/bin/sh
-    exec >&2
-    echo "For derivation $3:"
-    /run/current-system/sw/bin/diff -r "$1" "$2"
+```bash
+#!/bin/sh
+exec >&2
+echo "For derivation $3:"
+/run/current-system/sw/bin/diff -r "$1" "$2"
+```
 
 The diff hook is executed by the same user and group who ran the build.
 However, the diff hook does not have write access to the store path just
@@ -43,44 +47,55 @@ to the build command.
 If the build passes and is deterministic, Nix will exit with a status
 code of 0:
 
-    $ nix-build ./deterministic.nix -A stable
-    this derivation will be built:
-      /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
-    building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
-    /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
-    
-    $ nix-build ./deterministic.nix -A stable --check
-    checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
-    /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+```console
+$ nix-build ./deterministic.nix -A stable
+this derivation will be built:
+  /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
+building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+
+$ nix-build ./deterministic.nix -A stable --check
+checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+```
 
 If the build is not deterministic, Nix will exit with a status code of
 1:
 
-    $ nix-build ./deterministic.nix -A unstable
-    this derivation will be built:
-      /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
-    building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
-    /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
-    
-    $ nix-build ./deterministic.nix -A unstable --check
-    checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
-    error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
+```console
+$ nix-build ./deterministic.nix -A unstable
+this derivation will be built:
+  /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
+building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
+
+$ nix-build ./deterministic.nix -A unstable --check
+checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may
+not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
+```
 
 In the Nix daemon's log, we will now see:
 
-    For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv:
-    1c1
-    < 8108
-    ---
-    > 30204
+```
+For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv:
+1c1
+< 8108
+---
+> 30204
+```
 
 Using `--check` with `--keep-failed` will cause Nix to keep the second
 build's output in a special, `.check` path:
 
-    $ nix-build ./deterministic.nix -A unstable --check --keep-failed
-    checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
-    note: keeping build directory '/tmp/nix-build-unstable.drv-0'
-    error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check'
+```console
+$ nix-build ./deterministic.nix -A unstable --check --keep-failed
+checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+note: keeping build directory '/tmp/nix-build-unstable.drv-0'
+error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may
+not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
+from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check'
+```
 
 In particular, notice the
 `/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check` output. Nix
@@ -102,7 +117,8 @@ has copied the build results to that directory where you can examine it.
 already. If the derivation has not been built Nix will fail with the
 error:
 
-    error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible
+    error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' 
+    are not valid, so checking is not possible
 
 Run the build without `--check`, and then try with `--check` again.
 
@@ -130,10 +146,12 @@ reproducibly:
 
 An example output of this configuration:
 
-    $ nix-build ./test.nix -A unstable
-    this derivation will be built:
-      /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
-    building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
-    building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
-    output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
-    /nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
+```console
+$ nix-build ./test.nix -A unstable
+this derivation will be built:
+  /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
+building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
+building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
+output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
+/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
+```
diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
index 76a5380bf..c6966a50b 100644
--- a/doc/manual/src/advanced-topics/distributed-builds.md
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -11,12 +11,16 @@ To forward a build to a remote machine, it’s required that the remote
 machine is accessible via SSH and that it has Nix installed. You can
 test whether connecting to the remote Nix instance works, e.g.
 
-    $ nix ping-store --store ssh://mac
+```console
+$ nix ping-store --store ssh://mac
+```
 
 will try to connect to the machine named `mac`. It is possible to
 specify an SSH identity file as part of the remote store URI, e.g.
 
-    $ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key
+```console
+$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key
+```
 
 Since builds should be non-interactive, the key should not have a
 passphrase. Alternatively, you can load identities ahead of time into
@@ -24,8 +28,10 @@ passphrase. Alternatively, you can load identities ahead of time into
 
 If you get the error
 
-    bash: nix-store: command not found
-    error: cannot connect to 'mac'
+```console
+bash: nix-store: command not found
+error: cannot connect to 'mac'
+```
 
 then you need to ensure that the `PATH` of non-interactive login shells
 contains Nix.
@@ -43,21 +49,23 @@ the Nix configuration file. The former is convenient for testing. For
 example, the following command allows you to build a derivation for
 `x86_64-darwin` on a Linux machine:
 
-    $ uname
-    Linux
+```console
+$ uname
+Linux
     
-    $ nix build \
-      '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
-      --builders 'ssh://mac x86_64-darwin'
-    [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
-    
-    $ cat ./result
-    Darwin
+$ nix build \
+  '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+  --builders 'ssh://mac x86_64-darwin'
+[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
+
+$ cat ./result
+Darwin
+```
 
 It is possible to specify multiple builders separated by a semicolon or
 a newline, e.g.
 
-``` 
+```console
   --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
 ```
 
@@ -91,8 +99,10 @@ default, set it to `-`.
     the `requiredSystemFeatures` attribute, then Nix will only perform
     the derivation on a machine that has the specified features. For
     instance, the attribute
-    
-        requiredSystemFeatures = [ "kvm" ];
+
+    ```nix
+    requiredSystemFeatures = [ "kvm" ];
+    ```
     
     will cause the build to be performed on a machine that has the `kvm`
     feature.
@@ -111,11 +121,15 @@ For example, the machine specification
 specifies several machines that can perform `i686-linux` builds.
 However, `poochie` will only do builds that have the attribute
 
-    requiredSystemFeatures = [ "benchmark" ];
+```nix
+requiredSystemFeatures = [ "benchmark" ];
+```
 
 or
 
-    requiredSystemFeatures = [ "benchmark" "kvm" ];
+```nix
+requiredSystemFeatures = [ "benchmark" "kvm" ];
+```
 
 `itchy` cannot do builds that require `kvm`, but `scratchy` does support
 such builds. For regular builds, `itchy` will be preferred over
diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md
index 7b3ae58fb..bbdabed41 100644
--- a/doc/manual/src/advanced-topics/post-build-hook.md
+++ b/doc/manual/src/advanced-topics/post-build-hook.md
@@ -27,9 +27,11 @@ Use `nix-store --generate-binary-cache-key` to create our public and
 private signing keys. We will sign paths with the private key, and
 distribute the public key for verifying the authenticity of the paths.
 
-    # nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
-    # cat /etc/nix/key.public
-    example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+```console
+# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
+# cat /etc/nix/key.public
+example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+```
 
 Then, add the public key and the cache URL to your `nix.conf`'s
 `trusted-public-keys` and `substituters` options:
@@ -43,16 +45,18 @@ We will restart the Nix daemon in a later step.
 
 Write the following script to `/etc/nix/upload-to-cache.sh`:
 
-    #!/bin/sh
-    
-    set -eu
-    set -f # disable globbing
-    export IFS=' '
-    
-    echo "Signing paths" $OUT_PATHS
-    nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
-    echo "Uploading paths" $OUT_PATHS
-    exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
+```bash
+#!/bin/sh
+
+set -eu
+set -f # disable globbing
+export IFS=' '
+
+echo "Signing paths" $OUT_PATHS
+nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+echo "Uploading paths" $OUT_PATHS
+exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
+```
 
 > **Note**
 > 
@@ -65,7 +69,9 @@ Write the following script to `/etc/nix/upload-to-cache.sh`:
 
 Then make sure the hook program is executable by the `root` user:
 
-    # chmod +x /etc/nix/upload-to-cache.sh
+```console
+# chmod +x /etc/nix/upload-to-cache.sh
+```
 
 # Updating Nix Configuration
 
@@ -80,27 +86,33 @@ Then, restart the `nix-daemon`.
 
 Build any derivation, for example:
 
-    $ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)'
-    this derivation will be built:
-      /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
-    building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
-    running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
-    post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-    post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-    /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+```console
+$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)'
+this derivation will be built:
+  /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
+building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
+running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
+post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+```
 
 Then delete the path from the store, and try substituting it from the
 binary cache:
 
-    $ rm ./result
-    $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+```console
+$ rm ./result
+$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+```
 
 Now, copy the path back from the cache:
 
-    $ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
-    copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
-    warning: you did not specify '--add-root'; the result might be removed by the garbage collector
-    /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
+```console
+$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
+warning: you did not specify '--add-root'; the result might be removed by the garbage collector
+/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
+```
 
 # Conclusion
 
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
index 306ad23f5..028b15c9f 100644
--- a/doc/manual/src/command-ref/conf-file.md
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -70,11 +70,11 @@ The following settings are currently available:
     Note that trusted users are always allowed to connect.
 
   - `auto-optimise-store`  
-    If set to `true`, Nix automatically detects files in the store that
-    have identical contents, and replaces them with hard links to a
-    single copy. This saves disk space. If set to `false` (the default),
-    you can still run `nix-store
-                    --optimise` to get rid of duplicate files.
+    If set to `true`, Nix automatically detects files in the store
+    that have identical contents, and replaces them with hard links to
+    a single copy. This saves disk space. If set to `false` (the
+    default), you can still run `nix-store --optimise` to get rid of
+    duplicate files.
 
   - `builders`  
     A list of machines on which to perform builds.
@@ -214,11 +214,13 @@ The following settings are currently available:
     they have disappeared from their original URI. For example, given
     the default mirror `http://tarballs.nixos.org/`, when building the
     derivation
-    
-        builtins.fetchurl {
-          url = "https://example.org/foo-1.2.3.tar.xz";
-          sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-        }
+
+    ```nix
+    builtins.fetchurl {
+      url = "https://example.org/foo-1.2.3.tar.xz";
+      sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+    }
+    ```
     
     Nix will attempt to download this file from
     `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
@@ -233,8 +235,7 @@ The following settings are currently available:
     If set to `true` (the default), Nix will write the build log of a
     derivation (i.e. the standard output and error of its builder) to
     the directory `/nix/var/log/nix/drvs`. The build log can be
-    retrieved using the command `nix-store -l
-                    path`.
+    retrieved using the command `nix-store -l path`.
 
   - `keep-derivations`  
     If `true` (default), the garbage collector will keep the derivations
@@ -504,10 +505,9 @@ The following settings are currently available:
   - `secret-key-files`  
     A whitespace-separated list of files containing secret (private)
     keys. These are used to sign locally-built paths. They can be
-    generated using `nix-store
-                    --generate-binary-cache-key`. The corresponding public key can be
-    distributed to other users, who can add it to `trusted-public-keys`
-    in their `nix.conf`.
+    generated using `nix-store --generate-binary-cache-key`. The
+    corresponding public key can be distributed to other users, who
+    can add it to `trusted-public-keys` in their `nix.conf`.
 
   - `show-trace`  
     Causes Nix to print out a stack trace in case of Nix expression
@@ -601,18 +601,17 @@ The following settings are currently available:
 
   - `trusted-public-keys`  
     A whitespace-separated list of public keys. When paths are copied
-    from another Nix store (such as a binary cache), they must be signed
-    with one of these keys. For example:
+    from another Nix store (such as a binary cache), they must be
+    signed with one of these keys. For example:
     `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-                    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
+    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
 
   - `trusted-substituters`  
     A list of URLs of substituters, separated by whitespace. These are
     not used by default, but can be enabled by users of the Nix daemon
-    by specifying `--option
-                    substituters urls` on the command line. Unprivileged users are only
-    allowed to pass a subset of the URLs listed in `substituters` and
-    `trusted-substituters`.
+    by specifying `--option substituters urls` on the command
+    line. Unprivileged users are only allowed to pass a subset of the
+    URLs listed in `substituters` and `trusted-substituters`.
 
   - `trusted-users`  
     A list of names of users (separated by whitespace) that have
diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index e5fd45a7f..03016dba7 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -54,9 +54,11 @@ Most Nix commands interpret the following environment variables:
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
-    
-        $ mkdir /nix
-        $ mount -o bind /mnt/otherdisk/nix /nix
+
+    ```console
+    $ mkdir /nix
+    $ mount -o bind /mnt/otherdisk/nix /nix
+    ```
     
     Consult the mount 8 manual page for details.
 
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 026495c8d..4bcb8db40 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -66,20 +66,24 @@ The following common options are supported:
 
 # Examples
 
-    $ nix-build '<nixpkgs>' -A firefox
-    store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
-    /nix/store/d18hyl92g30l...-firefox-1.5.0.7
+```console
+$ nix-build '<nixpkgs>' -A firefox
+store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
+/nix/store/d18hyl92g30l...-firefox-1.5.0.7
 
-    $ ls -l result
-    lrwxrwxrwx  ...  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
+$ ls -l result
+lrwxrwxrwx  ...  result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7
 
-    $ ls ./result/bin/
-    firefox  firefox-config
+$ ls ./result/bin/
+firefox  firefox-config
+```
 
 If a derivation has multiple outputs, `nix-build` will build the default
 (first) output. You can also build all outputs:
 
-    $ nix-build '<nixpkgs>' -A openssl.all
+```console
+$ nix-build '<nixpkgs>' -A openssl.all
+```
 
 This will create a symlink for each output named `result-outputname`.
 The suffix is omitted if the output name is `out`. So if `openssl` has
@@ -87,19 +91,23 @@ outputs `out`, `bin` and `man`, `nix-build` will create symlinks
 `result`, `result-bin` and `result-man`. It’s also possible to build a
 specific output:
 
-    $ nix-build '<nixpkgs>' -A openssl.man
+```console
+$ nix-build '<nixpkgs>' -A openssl.man
+```
 
 This will create a symlink `result-man`.
 
 Build a Nix expression given on the command line:
 
-    $ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"'
-    $ cat ./result
-    bar
+```console
+$ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"'
+$ cat ./result
+bar
+```
 
 Build the GNU Hello package from the latest revision of the master
 branch of Nixpkgs:
 
-    $ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
-
-# Environment variables
+```console
+$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
+```
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index ea3a57e69..f0e205967 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -51,20 +51,24 @@ The list of subscribed channels is stored in `~/.nix-channels`.
 
 To subscribe to the Nixpkgs channel and install the GNU Hello package:
 
-    $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-    $ nix-channel --update
-    $ nix-env -iA nixpkgs.hello
+```console
+$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+$ nix-channel --update
+$ nix-env -iA nixpkgs.hello
+```
 
 You can revert channel updates using `--rollback`:
 
-    $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
-    "14.04.527.0e935f1"
+```console
+$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+"14.04.527.0e935f1"
 
-    $ nix-channel --rollback
-    switching from generation 483 to 482
+$ nix-channel --rollback
+switching from generation 483 to 482
 
-    $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
-    "14.04.526.dbadfad"
+$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+"14.04.526.dbadfad"
+```
 
 # Files
 
diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index 37587c7e1..62a6b7ca0 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -27,4 +27,6 @@ generations that were active at that point in time).
 To delete from the Nix store everything that is not used by the current
 generations of each profile, do
 
-    $ nix-collect-garbage -d
+```console
+$ nix-collect-garbage -d
+```
diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index 2a4558324..5ce320af7 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -73,11 +73,15 @@ and second to send the dump of those paths.  If this bothers you, use
 
 Copy Firefox with all its dependencies to a remote machine:
 
-    $ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)
+```console
+$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)
+```
 
 Copy Subversion from a remote machine and then install it into a user
 environment:
 
-    $ nix-copy-closure --from alice@itchy.labs \
-        /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
-    $ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+```console
+$ nix-copy-closure --from alice@itchy.labs \
+    /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+```
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index c5fcce316..ee838581b 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -127,10 +127,12 @@ have an effect.
     For example, if `~/.nix-defexpr` contains two files, `foo.nix` and
     `bar.nix`, then the default Nix expression will essentially be
 
-        {
-          foo = import ~/.nix-defexpr/foo.nix;
-          bar = import ~/.nix-defexpr/bar.nix;
-        }
+    ```nix
+    {
+      foo = import ~/.nix-defexpr/foo.nix;
+      bar = import ~/.nix-defexpr/bar.nix;
+    }
+    ```
 
     The file `manifest.nix` is always ignored. Subdirectories without a
     `default.nix` file are traversed recursively in search of more Nix
@@ -240,44 +242,60 @@ a number of possible ways:
 
 To install a specific version of `gcc` from the active Nix expression:
 
-    $ nix-env --install gcc-3.3.2
-    installing `gcc-3.3.2'
-    uninstalling `gcc-3.1'
+```console
+$ nix-env --install gcc-3.3.2
+installing `gcc-3.3.2'
+uninstalling `gcc-3.1'
+```
 
 Note the previously installed version is removed, since
 `--preserve-installed` was not specified.
 
 To install an arbitrary version:
 
-    $ nix-env --install gcc
-    installing `gcc-3.3.2'
+```console
+$ nix-env --install gcc
+installing `gcc-3.3.2'
+```
 
 To install using a specific attribute:
 
-    $ nix-env -i -A gcc40mips
-    $ nix-env -i -A xorg.xorgserver
+```console
+$ nix-env -i -A gcc40mips
+$ nix-env -i -A xorg.xorgserver
+```
 
 To install all derivations in the Nix expression `foo.nix`:
 
-    $ nix-env -f ~/foo.nix -i '.*'
+```console
+$ nix-env -f ~/foo.nix -i '.*'
+```
 
 To copy the store path with symbolic name `gcc` from another profile:
 
-    $ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc
+```console
+$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc
+```
 
 To install a specific store derivation (typically created by
 `nix-instantiate`):
 
-    $ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
+```console
+$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
+```
 
 To install a specific output path:
 
-    $ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3
+```console
+$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3
+```
 
 To install from a Nix expression specified on the command-line:
 
-    $ nix-env -f ./foo.nix -i -E \
-        'f: (f {system = "i686-linux";}).subversionWithJava'
+```console
+$ nix-env -f ./foo.nix -i -E \
+    'f: (f {system = "i686-linux";}).subversionWithJava'
+```
 
 I.e., this evaluates to `(f: (f {system =
 "i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting
@@ -286,17 +304,21 @@ function defined in `./foo.nix`.
 
 A dry-run tells you which paths will be downloaded or built from source:
 
-    $ nix-env -f '<nixpkgs>' -iA hello --dry-run
-    (dry run; not doing anything)
-    installing ‘hello-2.10’
-    this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
-      /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
-      ...
+```console
+$ nix-env -f '<nixpkgs>' -iA hello --dry-run
+(dry run; not doing anything)
+installing ‘hello-2.10’
+this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+  /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
+  ...
+```
 
 To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12
 channel:
 
-    $ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
+```console
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
+```
 
 # Operation `--upgrade`
 
@@ -353,18 +375,26 @@ For the other flags, see `--install`.
 
 ## Examples
 
-    $ nix-env --upgrade gcc
-    upgrading `gcc-3.3.1' to `gcc-3.4'
+```console
+$ nix-env --upgrade gcc
+upgrading `gcc-3.3.1' to `gcc-3.4'
+```
 
-    $ nix-env -u gcc-3.3.2 --always (switch to a specific version)
-    upgrading `gcc-3.4' to `gcc-3.3.2'
+```console
+$ nix-env -u gcc-3.3.2 --always (switch to a specific version)
+upgrading `gcc-3.4' to `gcc-3.3.2'
+```
 
-    $ nix-env --upgrade pan
-    (no upgrades available, so nothing happens)
+```console
+$ nix-env --upgrade pan
+(no upgrades available, so nothing happens)
+```
 
-    $ nix-env -u (try to upgrade everything)
-    upgrading `hello-2.1.2' to `hello-2.1.3'
-    upgrading `mozilla-1.2' to `mozilla-1.4'
+```console
+$ nix-env -u (try to upgrade everything)
+upgrading `hello-2.1.2' to `hello-2.1.3'
+upgrading `mozilla-1.2' to `mozilla-1.4'
+```
 
 ## Versions
 
@@ -416,8 +446,10 @@ designated by the symbolic names *drvnames* are removed.
 
 ## Examples
 
-    $ nix-env --uninstall gcc
-    $ nix-env -e '.*' (remove everything)
+```console
+$ nix-env --uninstall gcc
+$ nix-env -e '.*' (remove everything)
+```
 
 # Operation `--set`
 
@@ -435,7 +467,9 @@ that it contains exactly the specified derivation, and nothing else.
 The following updates a profile such that its current generation will
 contain just Firefox:
 
-    $ nix-env -p /nix/var/nix/profiles/browser --set firefox
+```console
+$ nix-env -p /nix/var/nix/profiles/browser --set firefox
+```
 
 # Operation `--set-flag`
 
@@ -473,37 +507,43 @@ environment build script:
 
 To prevent the currently installed Firefox from being upgraded:
 
-    $ nix-env --set-flag keep true firefox
+```console
+$ nix-env --set-flag keep true firefox
+```
 
 After this, `nix-env -u` will ignore Firefox.
 
 To disable the currently installed Firefox, then install a new Firefox
 while the old remains part of the profile:
 
-    $ nix-env -q
-    firefox-2.0.0.9 (the current one)
+```console
+$ nix-env -q
+firefox-2.0.0.9 (the current one)
 
-    $ nix-env --preserve-installed -i firefox-2.0.0.11
-    installing `firefox-2.0.0.11'
-    building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
-    collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox'
-      and `/nix/store/...-firefox-2.0.0.9/bin/firefox'.
-    (i.e., can’t have two active at the same time)
+$ nix-env --preserve-installed -i firefox-2.0.0.11
+installing `firefox-2.0.0.11'
+building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
+collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox'
+  and `/nix/store/...-firefox-2.0.0.9/bin/firefox'.
+(i.e., can’t have two active at the same time)
 
-    $ nix-env --set-flag active false firefox
-    setting flag on `firefox-2.0.0.9'
+$ nix-env --set-flag active false firefox
+setting flag on `firefox-2.0.0.9'
 
-    $ nix-env --preserve-installed -i firefox-2.0.0.11
-    installing `firefox-2.0.0.11'
+$ nix-env --preserve-installed -i firefox-2.0.0.11
+installing `firefox-2.0.0.11'
 
-    $ nix-env -q
-    firefox-2.0.0.11 (the enabled one)
-    firefox-2.0.0.9 (the disabled one)
+$ nix-env -q
+firefox-2.0.0.11 (the enabled one)
+firefox-2.0.0.9 (the disabled one)
+```
 
 To make files from `binutils` take precedence over files from `gcc`:
 
-    $ nix-env --set-flag priority 5 binutils
-    $ nix-env --set-flag priority 10 gcc
+```console
+$ nix-env --set-flag priority 5 binutils
+$ nix-env --set-flag priority 10 gcc
+```
 
 # Operation `--query`
 
@@ -633,66 +673,82 @@ derivation is shown unless `--no-name` is specified.
 
 To show installed packages:
 
-    $ nix-env -q
-    bison-1.875c
-    docbook-xml-4.2
-    firefox-1.0.4
-    MPlayer-1.0pre7
-    ORBit2-2.8.3
-    …
+```console
+$ nix-env -q
+bison-1.875c
+docbook-xml-4.2
+firefox-1.0.4
+MPlayer-1.0pre7
+ORBit2-2.8.3
+…
+```
 
 To show available packages:
 
-    $ nix-env -qa
-    firefox-1.0.7
-    GConf-2.4.0.1
-    MPlayer-1.0pre7
-    ORBit2-2.8.3
-    …
+```console
+$ nix-env -qa
+firefox-1.0.7
+GConf-2.4.0.1
+MPlayer-1.0pre7
+ORBit2-2.8.3
+…
+```
 
 To show the status of available packages:
 
-    $ nix-env -qas
-    -P- firefox-1.0.7   (not installed but present)
-    --S GConf-2.4.0.1   (not present, but there is a substitute for fast installation)
-    --S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!)
-    IP- ORBit2-2.8.3    (installed and by definition present)
-    …
+```console
+$ nix-env -qas
+-P- firefox-1.0.7   (not installed but present)
+--S GConf-2.4.0.1   (not present, but there is a substitute for fast installation)
+--S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!)
+IP- ORBit2-2.8.3    (installed and by definition present)
+…
+```
 
 To show available packages in the Nix expression `foo.nix`:
 
-    $ nix-env -f ./foo.nix -qa
-    foo-1.2.3
+```console
+$ nix-env -f ./foo.nix -qa
+foo-1.2.3
+```
 
 To compare installed versions to what’s available:
 
-    $ nix-env -qc
-    ...
-    acrobat-reader-7.0 - ?      (package is not available at all)
-    autoconf-2.59      = 2.59   (same version)
-    firefox-1.0.4      < 1.0.7  (a more recent version is available)
-    ...
+```console
+$ nix-env -qc
+...
+acrobat-reader-7.0 - ?      (package is not available at all)
+autoconf-2.59      = 2.59   (same version)
+firefox-1.0.4      < 1.0.7  (a more recent version is available)
+...
+```
 
 To show all packages with “`zip`” in the name:
 
-    $ nix-env -qa '.*zip.*'
-    bzip2-1.0.6
-    gzip-1.6
-    zip-3.0
-    …
+```console
+$ nix-env -qa '.*zip.*'
+bzip2-1.0.6
+gzip-1.6
+zip-3.0
+…
+```
 
 To show all packages with “`firefox`” or “`chromium`” in the name:
 
-    $ nix-env -qa '.*(firefox|chromium).*'
-    chromium-37.0.2062.94
-    chromium-beta-38.0.2125.24
-    firefox-32.0.3
-    firefox-with-plugins-13.0.1
-    …
+```console
+$ nix-env -qa '.*(firefox|chromium).*'
+chromium-37.0.2062.94
+chromium-beta-38.0.2125.24
+firefox-32.0.3
+firefox-with-plugins-13.0.1
+…
+```
 
 To show all packages in the latest revision of the Nixpkgs repository:
 
-    $ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
+```console
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
+```
 
 # Operation `--switch-profile`
 
@@ -707,7 +763,9 @@ the symlink `~/.nix-profile` is made to point to *path*.
 
 ## Examples
 
-    $ nix-env -S ~/my-profile
+```console
+$ nix-env -S ~/my-profile
+```
 
 # Operation `--list-generations`
 
@@ -724,11 +782,13 @@ generation, and indicates the current generation.
 
 ## Examples
 
-    $ nix-env --list-generations
-      95   2004-02-06 11:48:24
-      96   2004-02-06 11:49:01
-      97   2004-02-06 16:22:45
-      98   2004-02-06 16:24:33   (current)
+```console
+$ nix-env --list-generations
+  95   2004-02-06 11:48:24
+  96   2004-02-06 11:49:01
+  97   2004-02-06 16:22:45
+  98   2004-02-06 16:24:33   (current)
+```
 
 # Operation `--delete-generations`
 
@@ -750,13 +810,21 @@ generations is important to make garbage collection effective.
 
 ## Examples
 
-    $ nix-env --delete-generations 3 4 8
+```console
+$ nix-env --delete-generations 3 4 8
+```
 
-    $ nix-env --delete-generations +5
+```console
+$ nix-env --delete-generations +5
+```
 
-    $ nix-env --delete-generations 30d
+```console
+$ nix-env --delete-generations 30d
+```
 
-    $ nix-env -p other_profile --delete-generations old
+```console
+$ nix-env -p other_profile --delete-generations old
+```
 
 # Operation `--switch-generation`
 
@@ -776,8 +844,10 @@ Switching will fail if the specified generation does not exist.
 
 ## Examples
 
-    $ nix-env -G 42
-    switching from generation 50 to 42
+```console
+$ nix-env -G 42
+switching from generation 50 to 42
+```
 
 # Operation `--rollback`
 
@@ -794,11 +864,15 @@ generation, if it exists. It is just a convenience wrapper around
 
 ## Examples
 
-    $ nix-env --rollback
-    switching from generation 92 to 91
+```console
+$ nix-env --rollback
+switching from generation 92 to 91
+```
 
-    $ nix-env --rollback
-    error: no generation older than the current (91) exists
+```console
+$ nix-env --rollback
+error: no generation older than the current (91) exists
+```
 
 # Environment variables
 
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index edb331e1c..d3f91f8e9 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -61,38 +61,44 @@ md5sum`.
 
 Computing the same hash as `nix-prefetch-url`:
 
-    $ nix-prefetch-url file://<(echo test)
-    1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
-    $ nix-hash --type sha256 --flat --base32 <(echo test)
-    1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+```console
+$ nix-prefetch-url file://<(echo test)
+1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+$ nix-hash --type sha256 --flat --base32 <(echo test)
+1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+```
 
 Computing hashes:
 
-    $ mkdir test
-    $ echo "hello" > test/world
+```console
+$ mkdir test
+$ echo "hello" > test/world
 
-    $ nix-hash test/ (MD5 hash; default)
-    8179d3caeff1869b5ba1744e5a245c04
+$ nix-hash test/ (MD5 hash; default)
+8179d3caeff1869b5ba1744e5a245c04
 
-    $ nix-store --dump test/ | md5sum (for comparison)
-    8179d3caeff1869b5ba1744e5a245c04  -
+$ nix-store --dump test/ | md5sum (for comparison)
+8179d3caeff1869b5ba1744e5a245c04  -
 
-    $ nix-hash --type sha1 test/
-    e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
+$ nix-hash --type sha1 test/
+e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
 
-    $ nix-hash --type sha1 --base32 test/
-    nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
+$ nix-hash --type sha1 --base32 test/
+nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
 
-    $ nix-hash --type sha256 --flat test/
-    error: reading file `test/': Is a directory
+$ nix-hash --type sha256 --flat test/
+error: reading file `test/': Is a directory
 
-    $ nix-hash --type sha256 --flat test/world
-    5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03
+$ nix-hash --type sha256 --flat test/world
+5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03
+```
 
 Converting between hexadecimal and base-32:
 
-    $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
-    nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
+```console
+$ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
+nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
 
-    $ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
-    e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
+$ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4
+e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6
+```
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 2d6525e77..5b5ee0439 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -85,61 +85,77 @@ standard input.
 Instantiating store derivations from a Nix expression, and building them
 using `nix-store`:
 
-    $ nix-instantiate test.nix (instantiate)
-    /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
+```console
+$ nix-instantiate test.nix (instantiate)
+/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
 
-    $ nix-store -r $(nix-instantiate test.nix) (build)
-    ...
-    /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path)
+$ nix-store -r $(nix-instantiate test.nix) (build)
+...
+/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path)
 
-    $ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
-    dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
-    ...
+$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
+dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
+...
+```
 
 You can also give a Nix expression on the command line:
 
-    $ nix-instantiate -E 'with import <nixpkgs> { }; hello'
-    /nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv
+```console
+$ nix-instantiate -E 'with import <nixpkgs> { }; hello'
+/nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv
+```
 
 This is equivalent to:
 
-    $ nix-instantiate '<nixpkgs>' -A hello
+```console
+$ nix-instantiate '<nixpkgs>' -A hello
+```
 
 Parsing and evaluating Nix expressions:
 
-    $ nix-instantiate --parse -E '1 + 2'
-    1 + 2
+```console
+$ nix-instantiate --parse -E '1 + 2'
+1 + 2
+```
 
-    $ nix-instantiate --eval -E '1 + 2'
-    3
+```console
+$ nix-instantiate --eval -E '1 + 2'
+3
+```
 
-    $ nix-instantiate --eval --xml -E '1 + 2'
-    <?xml version='1.0' encoding='utf-8'?>
-    <expr>
-      <int value="3" />
-    </expr>
+```console
+$ nix-instantiate --eval --xml -E '1 + 2'
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <int value="3" />
+</expr>
+```
 
 The difference between non-strict and strict evaluation:
 
-    $ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }'
-    ...
-      <attr name="x">
-        <string value="foo" />
-      </attr>
-      <attr name="y">
-        <unevaluated />
-      </attr>
-    ...
+```console
+$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }'
+...
+  <attr name="x">
+    <string value="foo" />
+  </attr>
+  <attr name="y">
+    <unevaluated />
+  </attr>
+...
+```
 
 Note that `y` is left unevaluated (the XML representation doesn’t
 attempt to show non-normal forms).
 
-    $ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }'
-    ...
-      <attr name="x">
-        <string value="foo" />
-      </attr>
-      <attr name="y">
-        <string value="foo" />
-      </attr>
-    ...
+```console
+$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }'
+...
+  <attr name="x">
+    <string value="foo" />
+  </attr>
+  <attr name="y">
+    <string value="foo" />
+  </attr>
+...
+```
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index 688496969..1cd1063cd 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -59,13 +59,19 @@ Nix store is also printed.
 
 # Examples
 
-    $ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
-    0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
+```console
+$ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
+0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
+```
 
-    $ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
-    0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
-    /nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
+```console
+$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
+0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
+/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
+```
 
-    $ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz
-    079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7
-    /nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz
+```console
+$ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz
+079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7
+/nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz
+```
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 492351867..27826717b 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -39,10 +39,12 @@ after `$stdenv/setup` has been sourced. Since this hook is not executed
 by regular Nix builds, it allows you to perform initialisation specific
 to `nix-shell`. For example, the derivation attribute
 
-    shellHook =
-      ''
-        echo "Hello shell"
-      '';
+```nix
+shellHook =
+  ''
+    echo "Hello shell"
+  '';
+```
 
 will cause `nix-shell` to print `Hello shell`.
 
@@ -108,46 +110,58 @@ The following common options are supported:
 To build the dependencies of the package Pan, and start an interactive
 shell in which to build it:
 
-    $ nix-shell '<nixpkgs>' -A pan
-    [nix-shell]$ unpackPhase
-    [nix-shell]$ cd pan-*
-    [nix-shell]$ configurePhase
-    [nix-shell]$ buildPhase
-    [nix-shell]$ ./pan/gui/pan
+```shell
+$ nix-shell '<nixpkgs>' -A pan
+[nix-shell]$ unpackPhase
+[nix-shell]$ cd pan-*
+[nix-shell]$ configurePhase
+[nix-shell]$ buildPhase
+[nix-shell]$ ./pan/gui/pan
+```
 
 To clear the environment first, and do some additional automatic
 initialisation of the interactive shell:
 
-    $ nix-shell '<nixpkgs>' -A pan --pure \
-        --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
+```shell
+$ nix-shell '<nixpkgs>' -A pan --pure \
+    --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
+```
 
 Nix expressions can also be given on the command line using the `-E` and
 `-p` flags. For instance, the following starts a shell containing the
 packages `sqlite` and `libX11`:
 
-    $ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
+```shell
+$ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
+```
 
 A shorter way to do the same is:
 
-    $ nix-shell -p sqlite xorg.libX11
-    [nix-shell]$ echo $NIX_LDFLAGS
-    … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
+```shell
+$ nix-shell -p sqlite xorg.libX11
+[nix-shell]$ echo $NIX_LDFLAGS
+… -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
+```
 
 Note that `-p` accepts multiple full nix expressions that are valid in
 the `buildInputs = [ ... ]` shown above, not only package names. So the
 following is also legal:
 
-    $ nix-shell -p sqlite 'git.override { withManual = false; }'
+```shell
+$ nix-shell -p sqlite 'git.override { withManual = false; }'
+```
 
 The `-p` flag looks up Nixpkgs in the Nix search path. You can override
 it by passing `-I` or setting `NIX_PATH`. For example, the following
 gives you a shell containing the Pan package from a specific revision of
 Nixpkgs:
 
-    $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+```shell
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
-    [nix-shell:~]$ pan --version
-    Pan 0.139
+[nix-shell:~]$ pan --version
+Pan 0.139
+```
 
 # Use as a `#!`-interpreter
 
@@ -155,8 +169,10 @@ You can use `nix-shell` as a script interpreter to allow scripts written
 in arbitrary languages to obtain their own dependencies via Nix. This is
 done by starting the script with the following lines:
 
-    #! /usr/bin/env nix-shell
-    #! nix-shell -i real-interpreter -p packages
+```bash
+#! /usr/bin/env nix-shell
+#! nix-shell -i real-interpreter -p packages
+```
 
 where *real-interpreter* is the “real” script interpreter that will be
 invoked by `nix-shell` after it has obtained the dependencies and
@@ -170,39 +186,45 @@ because many operating systems only allow one argument in `#!` lines.
 For example, here is a Python script that depends on Python and the
 `prettytable` package:
 
-    #! /usr/bin/env nix-shell
-    #! nix-shell -i python -p python pythonPackages.prettytable
+```python
+#! /usr/bin/env nix-shell
+#! nix-shell -i python -p python pythonPackages.prettytable
 
-    import prettytable
+import prettytable
 
-    # Print a simple table.
-    t = prettytable.PrettyTable(["N", "N^2"])
-    for n in range(1, 10): t.add_row([n, n * n])
-    print t
+# Print a simple table.
+t = prettytable.PrettyTable(["N", "N^2"])
+for n in range(1, 10): t.add_row([n, n * n])
+print t
+```
 
 Similarly, the following is a Perl script that specifies that it
 requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages:
 
-    #! /usr/bin/env nix-shell
-    #! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
+```perl
+#! /usr/bin/env nix-shell
+#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
 
-    use HTML::TokeParser::Simple;
+use HTML::TokeParser::Simple;
 
-    # Fetch nixos.org and print all hrefs.
-    my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
+# Fetch nixos.org and print all hrefs.
+my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
 
-    while (my $token = $p->get_tag("a")) {
-        my $href = $token->get_attr("href");
-        print "$href\n" if $href;
-    }
+while (my $token = $p->get_tag("a")) {
+    my $href = $token->get_attr("href");
+    print "$href\n" if $href;
+}
+```
 
 Sometimes you need to pass a simple Nix expression to customize a
 package like Terraform:
 
-    #! /usr/bin/env nix-shell
-    #! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
+```bash
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
 
-    terraform apply
+terraform apply
+```
 
 > **Note**
 >
@@ -213,20 +235,22 @@ Finally, using the merging of multiple nix-shell shebangs the following
 Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable
 branch):
 
-    #! /usr/bin/env nix-shell
-    #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-    #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+```haskell
+#! /usr/bin/env nix-shell
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
 
-    import Network.HTTP
-    import Text.HTML.TagSoup
+import Network.HTTP
+import Text.HTML.TagSoup
 
-    -- Fetch nixos.org and print all hrefs.
-    main = do
-      resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
-      body <- getResponseBody resp
-      let tags = filter (isTagOpenName "a") $ parseTags body
-      let tags' = map (fromAttrib "href") tags
-      mapM_ putStrLn $ filter (/= "") tags'
+-- Fetch nixos.org and print all hrefs.
+main = do
+  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
+  body <- getResponseBody resp
+  let tags = filter (isTagOpenName "a") $ parseTags body
+  let tags' = map (fromAttrib "href") tags
+  mapM_ putStrLn $ filter (/= "") tags'
+```
 
 If you want to be even more precise, you can specify a specific revision
 of Nixpkgs:
@@ -237,12 +261,16 @@ The examples above all used `-p` to get dependencies from Nixpkgs. You
 can also use a Nix expression to build your own dependencies. For
 example, the Python example could have been written as:
 
-    #! /usr/bin/env nix-shell
-    #! nix-shell deps.nix -i python
+```python
+#! /usr/bin/env nix-shell
+#! nix-shell deps.nix -i python
+```
 
 where the file `deps.nix` in the same directory as the `#!`-script
 contains:
 
-    with import <nixpkgs> {};
+```nix
+with import <nixpkgs> {};
 
-    runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
+runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
+```
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 1842f8858..193d670c2 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -47,13 +47,15 @@ have an effect.
     The `--indirect` flag causes a uniquely named symlink to *path* to
     be stored in `/nix/var/nix/gcroots/auto/`. For instance,
 
-        $ nix-store --add-root /home/eelco/bla/result --indirect -r ...
+    ```console
+    $ nix-store --add-root /home/eelco/bla/result --indirect -r ...
 
-        $ ls -l /nix/var/nix/gcroots/auto
-        lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
+    $ ls -l /nix/var/nix/gcroots/auto
+    lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
 
-        $ ls -l /home/eelco/bla/result
-        lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10
+    $ ls -l /home/eelco/bla/result
+    lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10
+    ```
 
     Thus, when `/home/eelco/bla/result` is removed, the GC root in the
     `auto` directory becomes a dangling symlink and will be ignored by
@@ -157,14 +159,18 @@ or.
 This operation is typically used to build store derivations produced by
 [`nix-instantiate`](nix-instantiate.md):
 
-    $ nix-store -r $(nix-instantiate ./test.nix)
-    /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
+```console
+$ nix-store -r $(nix-instantiate ./test.nix)
+/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
+```
 
 This is essentially what [`nix-build`](nix-build.md) does.
 
 To test whether a previously-built derivation is deterministic:
 
-    $ nix-build '<nixpkgs>' -A hello --check -K
+```console
+$ nix-build '<nixpkgs>' -A hello --check -K
+```
 
 # Operation `--serve`
 
@@ -190,9 +196,11 @@ The following flags are available:
 To turn a host into a build server, the `authorized_keys` file can be
 used to provide build access to a given SSH public key:
 
-    $ cat <<EOF >>/root/.ssh/authorized_keys
-    command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA...
-    EOF
+```console
+$ cat <<EOF >>/root/.ssh/authorized_keys
+command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA...
+EOF
+```
 
 # Operation `--gc`
 
@@ -245,14 +253,18 @@ number of bytes that would be freed.
 
 To delete all unreachable paths, just do:
 
-    $ nix-store --gc
-    deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
-    ...
-    8825586 bytes freed (8.42 MiB)
+```console
+$ nix-store --gc
+deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
+...
+8825586 bytes freed (8.42 MiB)
+```
 
 To delete at least 100 MiBs of unreachable paths:
 
-    $ nix-store --gc --max-freed $((100 * 1024 * 1024))
+```console
+$ nix-store --gc --max-freed $((100 * 1024 * 1024))
+```
 
 # Operation `--delete`
 
@@ -274,9 +286,11 @@ paths in the store that refer to it (i.e., depend on it).
 
 ## Example
 
-    $ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4
-    0 bytes freed (0.00 MiB)
-    error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive
+```console
+$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4
+0 bytes freed (0.00 MiB)
+error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive
+```
 
 # Operation `--query`
 
@@ -407,18 +421,22 @@ symlink.
 Print the closure (runtime dependencies) of the `svn` program in the
 current user environment:
 
-    $ nix-store -qR $(which svn)
-    /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
-    /nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
-    ...
+```console
+$ nix-store -qR $(which svn)
+/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
+/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+...
+```
 
 Print the build-time dependencies of `svn`:
 
-    $ nix-store -qR $(nix-store -qd $(which svn))
-    /nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
-    /nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
-    /nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
-    ... lots of other paths ...
+```console
+$ nix-store -qR $(nix-store -qd $(which svn))
+/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
+/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
+/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
+... lots of other paths ...
+```
 
 The difference with the previous example is that we ask the closure of
 the derivation (`-qd`), not the closure of the output path that contains
@@ -426,29 +444,35 @@ the derivation (`-qd`), not the closure of the output path that contains
 
 Show the build-time dependencies as a tree:
 
-    $ nix-store -q --tree $(nix-store -qd $(which svn))
-    /nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
-    +---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
-    +---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
-    |   +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
-    |   +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
-    ...
+```console
+$ nix-store -q --tree $(nix-store -qd $(which svn))
+/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
++---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
++---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
+|   +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
+|   +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
+...
+```
 
 Show all paths that depend on the same OpenSSL library as `svn`:
 
-    $ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn)))
-    /nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0
-    /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
-    /nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3
-    /nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5
+```console
+$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn)))
+/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0
+/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
+/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3
+/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5
+```
 
 Show all paths that directly or indirectly depend on the Glibc (C
 library) used by `svn`:
 
-    $ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
-    /nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
-    /nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
-    ...
+```console
+$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
+/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
+/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
+...
+```
 
 Note that `ldd` is a command that prints out the dynamic libraries used
 by an ELF executable.
@@ -456,16 +480,20 @@ by an ELF executable.
 Make a picture of the runtime dependency graph of the current user
 environment:
 
-    $ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps
-    $ gv graph.ps
+```console
+$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps
+$ gv graph.ps
+```
 
 Show every garbage collector root that points to a store path that
 depends on `svn`:
 
-    $ nix-store -q --roots $(which svn)
-    /nix/var/nix/profiles/default-81-link
-    /nix/var/nix/profiles/default-82-link
-    /nix/var/nix/profiles/per-user/eelco/profile-97-link
+```console
+$ nix-store -q --roots $(which svn)
+/nix/var/nix/profiles/default-81-link
+/nix/var/nix/profiles/default-82-link
+/nix/var/nix/profiles/per-user/eelco/profile-97-link
+```
 
 # Operation `--add`
 
@@ -480,8 +508,10 @@ prints the resulting paths in the Nix store on standard output.
 
 ## Example
 
-    $ nix-store --add ./foo.c
-    /nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c
+```console
+$ nix-store --add ./foo.c
+/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c
+```
 
 # Operation `--add-fixed`
 
@@ -505,8 +535,10 @@ This operation has the following options:
 
 ## Example
 
-    $ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
-    /nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
+```console
+$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
+/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
+```
 
 # Operation `--verify`
 
@@ -554,7 +586,9 @@ path has changed, and 1 otherwise.
 
 To verify the integrity of the `svn` command and all its dependencies:
 
-    $ nix-store --verify-path $(nix-store -qR $(which svn))
+```console
+$ nix-store --verify-path $(nix-store -qR $(which svn))
+```
 
 # Operation `--repair-path`
 
@@ -578,14 +612,16 @@ substitutes are available, then repair is not possible.
 
 ## Example
 
-    $ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
-    path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified!
-      expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588',
-      got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4'
+```console
+$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
+path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified!
+  expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588',
+  got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4'
 
-    $ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
-    fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
-    …
+$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13
+fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'...
+…
+```
 
 # Operation `--dump`
 
@@ -651,11 +687,15 @@ a store path references other store paths that are missing in the target
 Nix store, the import will fail. To copy a whole closure, do something
 like:
 
-    $ nix-store --export $(nix-store -qR paths) > out
+```console
+$ nix-store --export $(nix-store -qR paths) > out
+```
 
 To import the whole closure again, run:
 
-    $ nix-store --import < out
+```console
+$ nix-store --import < out
+```
 
 # Operation `--import`
 
@@ -695,11 +735,13 @@ Use `-vv` or `-vvv` to get some progress indication.
 
 ## Example
 
-    $ nix-store --optimise
-    hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1'
-    ...
-    541838819 bytes (516.74 MiB) freed by hard-linking 54143 files;
-    there are 114486 files with equal contents out of 215894 files in total
+```console
+$ nix-store --optimise
+hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1'
+...
+541838819 bytes (516.74 MiB) freed by hard-linking 54143 files;
+there are 114486 files with equal contents out of 215894 files in total
+```
 
 # Operation `--read-log`
 
@@ -721,13 +763,15 @@ substitute, then the log is unavailable.
 
 ## Example
 
-    $ nix-store -l $(which ktorrent)
-    building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1
-    unpacking sources
-    unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz
-    ktorrent-2.2.1/
-    ktorrent-2.2.1/NEWS
-    ...
+```console
+$ nix-store -l $(which ktorrent)
+building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1
+unpacking sources
+unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz
+ktorrent-2.2.1/
+ktorrent-2.2.1/NEWS
+...
+```
 
 # Operation `--dump-db`
 
@@ -773,12 +817,14 @@ of the builder are placed in the variable `_args`.
 
 ## Example
 
-    $ nix-store --print-env $(nix-instantiate '<nixpkgs>' -A firefox)
-    …
-    export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2'
-    export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv'
-    export system; system='x86_64-linux'
-    export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh'
+```console
+$ nix-store --print-env $(nix-instantiate '<nixpkgs>' -A firefox)
+…
+export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2'
+export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv'
+export system; system='x86_64-linux'
+export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh'
+```
 
 # Operation `--generate-binary-cache-key`
 
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index ee8419fd2..65e72d000 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -158,11 +158,13 @@ Most Nix commands accept the following command-line options:
     
     For instance, the top-level `default.nix` in Nixpkgs is actually a
     function:
-    
-        { # The system (e.g., `i686-linux') for which to build the packages.
-          system ? builtins.currentSystem
-          ...
-        }: ...
+
+    ```nix
+    { # The system (e.g., `i686-linux') for which to build the packages.
+      system ? builtins.currentSystem
+      ...
+    }: ...
+    ```
     
     So if you call this Nix expression (e.g., when you do `nix-env -i
     pkgname`), the function will be called automatically using the
diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
index bfee28acf..31ebadda1 100644
--- a/doc/manual/src/expressions/advanced-attributes.md
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -6,7 +6,9 @@ Derivations can declare some infrequently used optional attributes.
     The optional attribute `allowedReferences` specifies a list of legal
     references (dependencies) of the output of the builder. For example,
     
-        allowedReferences = [];
+    ```nix
+    allowedReferences = [];
+    ```
     
     enforces that the output of a derivation cannot have any runtime
     dependencies on its inputs. To allow an output to have a runtime
@@ -20,7 +22,9 @@ Derivations can declare some infrequently used optional attributes.
     the legal requisites of the whole closure, so all the dependencies
     recursively. For example,
     
-        allowedRequisites = [ foobar ];
+    ```nix
+    allowedRequisites = [ foobar ];
+    ```
     
     enforces that the output of a derivation cannot have any other
     runtime dependency than `foobar`, and in addition it enforces that
@@ -31,7 +35,9 @@ Derivations can declare some infrequently used optional attributes.
     illegal references (dependencies) of the output of the builder. For
     example,
     
-        disallowedReferences = [ foo ];
+    ```nix
+    disallowedReferences = [ foo ];
+    ```
     
     enforces that the output of a derivation cannot have a direct
     runtime dependencies on the derivation `foo`.
@@ -41,7 +47,9 @@ Derivations can declare some infrequently used optional attributes.
     specifies illegal requisites for the whole closure, so all the
     dependencies recursively. For example,
     
-        disallowedRequisites = [ foobar ];
+    ```nix
+    disallowedRequisites = [ foobar ];
+    ```
     
     enforces that the output of a derivation cannot have any runtime
     dependency on `foobar` or any other derivation depending recursively
@@ -50,20 +58,20 @@ Derivations can declare some infrequently used optional attributes.
   - `exportReferencesGraph`  
     This attribute allows builders access to the references graph of
     their inputs. The attribute is a list of inputs in the Nix store
-    whose references graph the builder needs to know. The value of this
-    attribute should be a list of pairs `[ name1
-                    path1 name2
-                    path2 ...
-                    ]`. The references graph of each *pathN* will be stored in a text
-    file *nameN* in the temporary build directory. The text files have
-    the format used by `nix-store
-                    --register-validity` (with the deriver fields left empty). For
-    example, when the following derivation is built:
+    whose references graph the builder needs to know. The value of
+    this attribute should be a list of pairs `[ name1 path1 name2
+    path2 ...  ]`. The references graph of each *pathN* will be stored
+    in a text file *nameN* in the temporary build directory. The text
+    files have the format used by `nix-store --register-validity`
+    (with the deriver fields left empty). For example, when the
+    following derivation is built:
     
-        derivation {
-          ...
-          exportReferencesGraph = [ "libfoo-graph" libfoo ];
-        };
+    ```nix
+    derivation {
+      ...
+      exportReferencesGraph = [ "libfoo-graph" libfoo ];
+    };
+    ```
     
     the references graph of `libfoo` is placed in the file
     `libfoo-graph` in the temporary build directory.
@@ -84,7 +92,9 @@ Derivations can declare some infrequently used optional attributes.
     environment variables to be passed unmodified. For example,
     `fetchurl` in Nixpkgs has the line
     
-        impureEnvVars = [ "http_proxy" "https_proxy" ... ];
+    ```nix
+    impureEnvVars = [ "http_proxy" "https_proxy" ... ];
+    ```
     
     to make it use the proxy server configuration specified by the user
     in the environment variables `http_proxy` and friends.
@@ -116,19 +126,23 @@ Derivations can declare some infrequently used optional attributes.
     been modified, the caller must also specify a cryptographic hash of
     the file. For example,
     
-        fetchurl {
-          url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
-          sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-        }
+    ```nix
+    fetchurl {
+      url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
+      sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+    }
+    ```
     
     It sometimes happens that the URL of the file changes, e.g., because
     servers are reorganised or no longer available. We then must update
     the call to `fetchurl`, e.g.,
     
-        fetchurl {
-          url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-          sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-        }
+    ```nix
+    fetchurl {
+      url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+      sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+    }
+    ```
     
     If a `fetchurl` derivation was treated like a normal derivation, the
     output paths of the derivation and *all derivations depending on it*
@@ -147,23 +161,25 @@ Derivations can declare some infrequently used optional attributes.
     As an example, here is the (simplified) Nix expression for
     `fetchurl`:
     
-        { stdenv, curl }: # The curl program is used for downloading.
-        
-        { url, sha256 }:
-        
-        stdenv.mkDerivation {
-          name = baseNameOf (toString url);
-          builder = ./builder.sh;
-          buildInputs = [ curl ];
-        
-          # This is a fixed-output derivation; the output must be a regular
-          # file with SHA256 hash sha256.
-          outputHashMode = "flat";
-          outputHashAlgo = "sha256";
-          outputHash = sha256;
-        
-          inherit url;
-        }
+    ```nix
+    { stdenv, curl }: # The curl program is used for downloading.
+
+    { url, sha256 }:
+
+    stdenv.mkDerivation {
+      name = baseNameOf (toString url);
+      builder = ./builder.sh;
+      buildInputs = [ curl ];
+
+      # This is a fixed-output derivation; the output must be a regular
+      # file with SHA256 hash sha256.
+      outputHashMode = "flat";
+      outputHashAlgo = "sha256";
+      outputHash = sha256;
+
+      inherit url;
+    }
+    ```
     
     The `outputHashAlgo` attribute specifies the hash algorithm used to
     compute the hash. It can currently be `"sha1"`, `"sha256"` or
@@ -196,21 +212,19 @@ Derivations can declare some infrequently used optional attributes.
     A list of names of attributes that should be passed via files rather
     than environment variables. For example, if you have
     
-    ``` 
+    ```nix
     passAsFile = ["big"];
     big = "a very long string";
-        
     ```
     
-    then when the builder runs, the environment variable `bigPath` will
-    contain the absolute path to a temporary file containing `a very
-    long
-                    string`. That is, for any attribute *x* listed in `passAsFile`, Nix
-    will pass an environment variable `xPath` holding the path of the
-    file containing the value of attribute *x*. This is useful when you
-    need to pass large strings to a builder, since most operating
-    systems impose a limit on the size of the environment (typically, a
-    few hundred kilobyte).
+    then when the builder runs, the environment variable `bigPath`
+    will contain the absolute path to a temporary file containing `a
+    very long string`. That is, for any attribute *x* listed in
+    `passAsFile`, Nix will pass an environment variable `xPath`
+    holding the path of the file containing the value of attribute
+    *x*. This is useful when you need to pass large strings to a
+    builder, since most operating systems impose a limit on the size
+    of the environment (typically, a few hundred kilobyte).
 
   - `preferLocalBuild`  
     If this attribute is set to `true` and [distributed building is
diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md
index 2956f98f1..12198c879 100644
--- a/doc/manual/src/expressions/arguments-variables.md
+++ b/doc/manual/src/expressions/arguments-variables.md
@@ -8,25 +8,27 @@ packages are imported and called with the appropriate arguments. Here
 are some fragments of `all-packages.nix`, with annotations of what
 they mean:
 
-    ...
-    
-    rec { ①
-    
-      hello = import ../applications/misc/hello/ex-1 ② { ③
-        inherit fetchurl stdenv perl;
-      };
-    
-      perl = import ../development/interpreters/perl { ④
-        inherit fetchurl stdenv;
-      };
-    
-      fetchurl = import ../build-support/fetchurl {
-        inherit stdenv; ...
-      };
-    
-      stdenv = ...;
-    
-    }
+```nix
+...
+
+rec { ①
+
+  hello = import ../applications/misc/hello/ex-1 ② { ③
+    inherit fetchurl stdenv perl;
+  };
+
+  perl = import ../development/interpreters/perl { ④
+    inherit fetchurl stdenv;
+  };
+
+  fetchurl = import ../build-support/fetchurl {
+    inherit stdenv; ...
+  };
+
+  stdenv = ...;
+
+}
+```
 
 1.  This file defines a set of attributes, all of which are concrete
     derivations (i.e., not functions). In fact, we define a *mutually
@@ -64,11 +66,15 @@ they mean:
     > calls a function, filling in any missing arguments by passing the
     > corresponding attribute from the Nixpkgs set, like this:
     > 
-    >     hello = callPackage ../applications/misc/hello/ex-1 { };
+    > ```nix
+    > hello = callPackage ../applications/misc/hello/ex-1 { };
+    > ```
     > 
     > If necessary, you can set or override arguments:
     > 
-    >     hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; };
+    > ```nix
+    > hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; };
+    > ```
 
 4.  Likewise, we have to instantiate Perl, `fetchurl`, and the standard
     environment.
diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md
index e0dda56f8..b1eacae88 100644
--- a/doc/manual/src/expressions/build-script.md
+++ b/doc/manual/src/expressions/build-script.md
@@ -3,15 +3,17 @@
 Here is the builder referenced from Hello's Nix expression (stored in
 `pkgs/applications/misc/hello/ex-1/builder.sh`):
 
-    source $stdenv/setup ①
-    
-    PATH=$perl/bin:$PATH ②
-    
-    tar xvfz $src ③
-    cd hello-*
-    ./configure --prefix=$out ④
-    make ⑤
-    make install
+```bash
+source $stdenv/setup ①
+
+PATH=$perl/bin:$PATH ②
+
+tar xvfz $src ③
+cd hello-*
+./configure --prefix=$out ④
+make ⑤
+make install
+```
 
 The builder can actually be made a lot shorter by using the *generic
 builder* functions provided by `stdenv`, but here we write out the build
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index 22f133c33..7c4a62f54 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -51,7 +51,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
     You can use `builtins` to test for the availability of features in
     the Nix installation, e.g.,
     
-        if builtins ? getEnv then builtins.getEnv "PATH" else ""
+    ```nix
+    if builtins ? getEnv then builtins.getEnv "PATH" else ""
+    ```
     
     This allows a Nix expression to fall back gracefully on older Nix
     installations that don’t have the desired built-in function.
@@ -114,9 +116,11 @@ For instance, `derivation` is also available as `builtins.derivation`.
     function is to obtain external Nix expression dependencies, such as
     a particular version of Nixpkgs, e.g.
     
-        with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
-        
-        stdenv.mkDerivation { … }
+    ```nix
+    with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
+
+    stdenv.mkDerivation { … }
+    ```
     
     The fetched tarball is cached for a certain amount of time (1 hour
     by default) in `~/.cache/nix/tarballs/`. You can change the cache
@@ -124,19 +128,21 @@ For instance, `derivation` is also available as `builtins.derivation`.
     of seconds` or in the Nix configuration file with this option: ` 
     number of seconds to cache `.
     
-    Note that when obtaining the hash with ` nix-prefetch-url
-                     ` the option `--unpack` is required.
+    Note that when obtaining the hash with ` nix-prefetch-url ` the
+    option `--unpack` is required.
     
     This function can also verify the contents against a hash. In that
     case, the function takes a set instead of a URL. The set requires
     the attribute `url` and the attribute `sha256`, e.g.
     
-        with import (fetchTarball {
-          url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
-          sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
-        }) {};
-        
-        stdenv.mkDerivation { … }
+    ```nix
+    with import (fetchTarball {
+      url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
+      sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
+    }) {};
+
+    stdenv.mkDerivation { … }
+    ```
     
     This function is not available if [restricted evaluation
     mode](../command-ref/conf-file.md) is enabled.
@@ -172,18 +178,22 @@ For instance, `derivation` is also available as `builtins.derivation`.
     
       - To fetch a private repository over SSH:
         
-            builtins.fetchGit {
-              url = "git@github.com:my-secret/repository.git";
-              ref = "master";
-              rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
-            }
+        ```nix
+        builtins.fetchGit {
+          url = "git@github.com:my-secret/repository.git";
+          ref = "master";
+          rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+        }
+        ```
     
       - To fetch an arbitrary reference:
         
-            builtins.fetchGit {
-              url = "https://github.com/NixOS/nix.git";
-              ref = "refs/heads/0.5-release";
-            }
+        ```nix
+        builtins.fetchGit {
+          url = "https://github.com/NixOS/nix.git";
+          ref = "refs/heads/0.5-release";
+        }
+        ```
     
       - If the revision you're looking for is in the default branch of
         the git repository you don't strictly need to specify the branch
@@ -193,11 +203,13 @@ For instance, `derivation` is also available as `builtins.derivation`.
         branch for the non-default branch you will need to specify the
         the `ref` attribute as well.
         
-            builtins.fetchGit {
-              url = "https://github.com/nixos/nix.git";
-              rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-              ref = "1.11-maintenance";
-            }
+        ```nix
+        builtins.fetchGit {
+          url = "https://github.com/nixos/nix.git";
+          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+          ref = "1.11-maintenance";
+        }
+        ```
         
         > **Note**
         > 
@@ -211,24 +223,30 @@ For instance, `derivation` is also available as `builtins.derivation`.
       - If the revision you're looking for is in the default branch of
         the git repository you may omit the `ref` attribute.
         
-            builtins.fetchGit {
-              url = "https://github.com/nixos/nix.git";
-              rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-            }
+        ```nix
+        builtins.fetchGit {
+          url = "https://github.com/nixos/nix.git";
+          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+        }
+        ```
     
       - To fetch a specific tag:
         
-            builtins.fetchGit {
-              url = "https://github.com/nixos/nix.git";
-              ref = "refs/tags/1.9";
-            }
+        ```nix
+        builtins.fetchGit {
+          url = "https://github.com/nixos/nix.git";
+          ref = "refs/tags/1.9";
+        }
+        ```
     
       - To fetch the latest version of a remote branch:
         
-            builtins.fetchGit {
-              url = "ssh://git@github.com/nixos/nix.git";
-              ref = "master";
-            }
+        ```nix
+        builtins.fetchGit {
+          url = "ssh://git@github.com/nixos/nix.git";
+          ref = "master";
+        }
+        ```
         
         > **Note**
         > 
@@ -248,10 +266,12 @@ For instance, `derivation` is also available as `builtins.derivation`.
     filtering certain files. For instance, suppose that you want to use
     the directory `source-dir` as an input to a Nix expression, e.g.
     
-        stdenv.mkDerivation {
-          ...
-          src = ./source-dir;
-        }
+    ```nix
+    stdenv.mkDerivation {
+      ...
+      src = ./source-dir;
+    }
+    ```
     
     However, if `source-dir` is a Subversion working copy, then all
     those annoying `.svn` subdirectories will also be copied to the
@@ -259,10 +279,10 @@ For instance, `derivation` is also available as `builtins.derivation`.
     causing lots of spurious rebuilds. With `filterSource` you can
     filter out the `.svn` directories:
     
-    ``` 
-      src = builtins.filterSource
-        (path: type: type != "directory" || baseNameOf path != ".svn")
-        ./source-dir;
+    ```nix
+    src = builtins.filterSource
+      (path: type: type != "directory" || baseNameOf path != ".svn")
+      ./source-dir;
     ```
     
     Thus, the first argument *e1* must be a predicate function that is
@@ -279,10 +299,10 @@ For instance, `derivation` is also available as `builtins.derivation`.
 
   - `builtins.foldl’` *op* *nul* *list*  
     Reduce a list by applying a binary operator, from left to right,
-    e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op
-                    (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e.,
-    its arguments are evaluated first. For example, `foldl’ (x: y: x +
-    y) 0 [1 2 3]` evaluates to 6.
+    e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
+    ...`. The operator is applied strictly, i.e., its arguments are
+    evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]`
+    evaluates to 6.
 
   - `builtins.functionArgs` *f*  
     Return a set containing the names of the formal arguments expected
@@ -298,16 +318,19 @@ For instance, `derivation` is also available as `builtins.derivation`.
   - `builtins.fromJSON` *e*  
     Convert a JSON string to a Nix value. For example,
     
-        builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
+    ```nix
+    builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
+    ```
     
-    returns the value `{ x = [ 1 2 3 ]; y = null;
-                    }`.
+    returns the value `{ x = [ 1 2 3 ]; y = null; }`.
 
   - `builtins.genList` *generator* *length*  
     Generate list of size *length*, with each element *i* equal to the
     value returned by *generator* `i`. For example,
     
-        builtins.genList (x: x * x) 5
+    ```nix
+    builtins.genList (x: x * x) 5
+    ```
     
     returns the list `[ 0 1 4 9 16 ]`.
 
@@ -369,26 +392,34 @@ For instance, `derivation` is also available as `builtins.derivation`.
     variables that are in scope at the call site. For instance, if you
     have a calling expression
     
-        rec {
-          x = 123;
-          y = import ./foo.nix;
-        }
+    ```nix
+    rec {
+      x = 123;
+      y = import ./foo.nix;
+    }
+    ```
     
     then the following `foo.nix` will give an error:
     
-        x + 456
+    ```nix
+    x + 456
+    ```
     
     since `x` is not in scope in `foo.nix`. If you want `x` to be
     available in `foo.nix`, you should pass it as a function argument:
     
-        rec {
-          x = 123;
-          y = import ./foo.nix x;
-        }
+    ```nix
+    rec {
+      x = 123;
+      y = import ./foo.nix x;
+    }
+    ```
     
     and
     
-        x: x + 456
+    ```nix
+    x: x + 456
+    ```
     
     (The function argument doesn’t have to be called `x` in `foo.nix`;
     any name would work.)
@@ -442,23 +473,28 @@ For instance, `derivation` is also available as `builtins.derivation`.
     string-valued attribute `name` specifying the name of the attribute,
     and an attribute `value` specifying its value. Example:
     
-        builtins.listToAttrs
-          [ { name = "foo"; value = 123; }
-            { name = "bar"; value = 456; }
-          ]
+    ```nix
+    builtins.listToAttrs
+      [ { name = "foo"; value = 123; }
+        { name = "bar"; value = 456; }
+      ]
+    ```
     
     evaluates to
     
-        { foo = 123; bar = 456; }
+    ```nix
+    { foo = 123; bar = 456; }
+    ```
 
   - `map` *f* *list*; `builtins.map` *f* *list*  
     Apply the function *f* to each element in the list *list*. For
     example,
     
-        map (x: "foo" + x) [ "bar" "bla" "abc" ]
+    ```nix
+    map (x: "foo" + x) [ "bar" "bla" "abc" ]
+    ```
     
-    evaluates to `[ "foobar" "foobla" "fooabc"
-                    ]`.
+    evaluates to `[ "foobar" "foobla" "fooabc" ]`.
 
   - `builtins.match` *regex* *str*  
     Returns a list if the [extended POSIX regular
@@ -466,19 +502,27 @@ For instance, `derivation` is also available as `builtins.derivation`.
     *regex* matches *str* precisely, otherwise returns `null`. Each item
     in the list is a regex group.
     
-        builtins.match "ab" "abc"
+    ```nix
+    builtins.match "ab" "abc"
+    ```
     
     Evaluates to `null`.
     
-        builtins.match "abc" "abc"
+    ```nix
+    builtins.match "abc" "abc"
+    ```
     
     Evaluates to `[ ]`.
     
-        builtins.match "a(b)(c)" "abc"
+    ```nix
+    builtins.match "a(b)(c)" "abc"
+    ```
     
     Evaluates to `[ "b" "c" ]`.
     
-        builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
+    ```nix
+    builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
+    ```
     
     Evaluates to `[ "foo" ]`.
 
@@ -534,11 +578,12 @@ For instance, `derivation` is also available as `builtins.derivation`.
   - `builtins.readDir` *path*  
     Return the contents of the directory *path* as a set mapping
     directory entries to the corresponding file type. For instance, if
-    directory `A` contains a regular file `B` and another directory `C`,
-    then `builtins.readDir
-                    ./A` will return the set
+    directory `A` contains a regular file `B` and another directory
+    `C`, then `builtins.readDir ./A` will return the set
     
-        { B = "regular"; C = "directory"; }
+    ```nix
+    { B = "regular"; C = "directory"; }
+    ```
     
     The possible values for the file type are `"regular"`,
     `"directory"`, `"symlink"` and `"unknown"`.
@@ -550,7 +595,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
     Remove the attributes listed in *list* from *set*. The attributes
     don’t have to exist in *set*. For instance,
     
-        removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]
+    ```nix
+    removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]
+    ```
     
     evaluates to `{ y = 2; }`.
 
@@ -558,7 +605,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
     Given string *s*, replace every occurrence of the strings in *from*
     with the corresponding string in *to*. For example,
     
-        builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
+    ```nix
+    builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
+    ```
     
     evaluates to `"fabir"`.
 
@@ -572,10 +621,11 @@ For instance, `derivation` is also available as `builtins.derivation`.
     if the first element is less than the second, and `false` otherwise.
     For example,
     
-        builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
+    ```nix
+    builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
+    ```
     
-    produces the list `[ 42 77 147 249 483 526
-                    ]`.
+    produces the list `[ 42 77 147 249 483 526 ]`.
     
     This is a stable sort: it preserves the relative order of elements
     deemed equal by the comparator.
@@ -587,19 +637,27 @@ For instance, `derivation` is also available as `builtins.derivation`.
     *regex* matches of *str*. Each item in the lists of matched
     sequences is a regex group.
     
-        builtins.split "(a)b" "abc"
+    ```nix
+    builtins.split "(a)b" "abc"
+    ```
     
     Evaluates to `[ "" [ "a" ] "c" ]`.
     
-        builtins.split "([ac])" "abc"
+    ```nix
+    builtins.split "([ac])" "abc"
+    ```
     
     Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`.
     
-        builtins.split "(a)|(c)" "abc"
+    ```nix
+    builtins.split "(a)|(c)" "abc"
+    ```
     
     Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`.
     
-        builtins.split "([[:upper:]]+)" "  FOO   "
+    ```nix
+    builtins.split "([[:upper:]]+)" "  FOO   "
+    ```
     
     Evaluates to `[ " " [ "FOO" ] " " ]`.
 
@@ -623,7 +681,9 @@ For instance, `derivation` is also available as `builtins.derivation`.
     substring up to the end of the string is returned. *start* must be
     non-negative. For example,
     
-        builtins.substring 0 3 "nixos"
+    ```nix
+    builtins.substring 0 3 "nixos"
+    ```
     
     evaluates to `"nix"`.
 
@@ -645,45 +705,47 @@ For instance, `derivation` is also available as `builtins.derivation`.
     “inline”. For instance, the following Nix expression combines the
     [Nix expression for GNU Hello](expression-syntax.md) and its
     [build script](build-script.md) into one file:
-    
-        { stdenv, fetchurl, perl }:
-        
-        stdenv.mkDerivation {
-          name = "hello-2.1.1";
-        
-          builder = builtins.toFile "builder.sh" "
-            source $stdenv/setup
-        
-            PATH=$perl/bin:$PATH
-        
-            tar xvfz $src
-            cd hello-*
-            ./configure --prefix=$out
-            make
-            make install
-          ";
-        
-          src = fetchurl {
-            url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-            sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-          };
-          inherit perl;
-        }
+
+    ```nix
+    { stdenv, fetchurl, perl }:
+
+    stdenv.mkDerivation {
+      name = "hello-2.1.1";
+
+      builder = builtins.toFile "builder.sh" "
+        source $stdenv/setup
+
+        PATH=$perl/bin:$PATH
+
+        tar xvfz $src
+        cd hello-*
+        ./configure --prefix=$out
+        make
+        make install
+      ";
+
+      src = fetchurl {
+        url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+        sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+      };
+      inherit perl;
+    }
+    ```
     
     It is even possible for one file to refer to another, e.g.,
     
-    ``` 
-      builder = let
-        configFile = builtins.toFile "foo.conf" "
-          # This is some dummy configuration file.
-          ...
-        ";
-      in builtins.toFile "builder.sh" "
-        source $stdenv/setup
+    ```nix
+    builder = let
+      configFile = builtins.toFile "foo.conf" "
+        # This is some dummy configuration file.
         ...
-        cp ${configFile} $out/etc/foo.conf
       ";
-    ```
+    in builtins.toFile "builder.sh" "
+      source $stdenv/setup
+      ...
+      cp ${configFile} $out/etc/foo.conf
+    ";
+  ```
     
     Note that `${configFile}` is an
     [antiquotation](language-values.md), so the result of the
@@ -694,10 +756,12 @@ For instance, `derivation` is also available as `builtins.derivation`.
     It is however *not* allowed to have files mutually referring to each
     other, like so:
     
-        let
-          foo = builtins.toFile "foo" "...${bar}...";
-          bar = builtins.toFile "bar" "...${foo}...";
-        in foo
+    ```nix
+    let
+      foo = builtins.toFile "foo" "...${bar}...";
+      bar = builtins.toFile "bar" "...${foo}...";
+    in foo
+    ```
     
     This is not allowed because it would cause a cyclic dependency in
     the computation of the cryptographic hashes for `foo` and `bar`.
@@ -744,40 +808,42 @@ For instance, `derivation` is also available as `builtins.derivation`.
     
     Here is an example where this is the case:
     
-        { stdenv, fetchurl, libxslt, jira, uberwiki }:
-        
-        stdenv.mkDerivation (rec {
-          name = "web-server";
-        
-          buildInputs = [ libxslt ];
-        
-          builder = builtins.toFile "builder.sh" "
-            source $stdenv/setup
-            mkdir $out
-            echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ① 
-          ";
-        
-          stylesheet = builtins.toFile "stylesheet.xsl" ② 
-           "<?xml version='1.0' encoding='UTF-8'?>
-            <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
-              <xsl:template match='/'>
-                <Configure>
-                  <xsl:for-each select='/expr/list/attrs'>
-                    <Call name='addWebApplication'>
-                      <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg>
-                      <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg>
-                    </Call>
-                  </xsl:for-each>
-                </Configure>
-              </xsl:template>
-            </xsl:stylesheet>
-          ";
-        
-          servlets = builtins.toXML [ ③ 
-            { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
-            { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
-          ];
-        })
+    ```nix
+    { stdenv, fetchurl, libxslt, jira, uberwiki }:
+
+    stdenv.mkDerivation (rec {
+      name = "web-server";
+
+      buildInputs = [ libxslt ];
+
+      builder = builtins.toFile "builder.sh" "
+        source $stdenv/setup
+        mkdir $out
+        echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ① 
+      ";
+
+      stylesheet = builtins.toFile "stylesheet.xsl" ② 
+       "<?xml version='1.0' encoding='UTF-8'?>
+        <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
+          <xsl:template match='/'>
+            <Configure>
+              <xsl:for-each select='/expr/list/attrs'>
+                <Call name='addWebApplication'>
+                  <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg>
+                  <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg>
+                </Call>
+              </xsl:for-each>
+            </Configure>
+          </xsl:template>
+        </xsl:stylesheet>
+      ";
+
+      servlets = builtins.toXML [ ③ 
+        { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
+        { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
+      ];
+    })
+    ```
     
     The builder is supposed to generate the configuration file for a
     [Jetty servlet container](http://jetty.mortbay.org/). A servlet
@@ -796,27 +862,29 @@ For instance, `derivation` is also available as `builtins.derivation`.
     configuration file for the Jetty server. The XML representation
     produced at point ③ by `toXML` is as follows:
     
-        <?xml version='1.0' encoding='utf-8'?>
-        <expr>
-          <list>
-            <attrs>
-              <attr name="path">
-                <string value="/bugtracker" />
-              </attr>
-              <attr name="war">
-                <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" />
-              </attr>
-            </attrs>
-            <attrs>
-              <attr name="path">
-                <string value="/wiki" />
-              </attr>
-              <attr name="war">
-                <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" />
-              </attr>
-            </attrs>
-          </list>
-        </expr>
+    ```xml
+    <?xml version='1.0' encoding='utf-8'?>
+    <expr>
+      <list>
+        <attrs>
+          <attr name="path">
+            <string value="/bugtracker" />
+          </attr>
+          <attr name="war">
+            <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" />
+          </attr>
+        </attrs>
+        <attrs>
+          <attr name="path">
+            <string value="/wiki" />
+          </attr>
+          <attr name="war">
+            <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" />
+          </attr>
+        </attrs>
+      </list>
+    </expr>
+    ```
     
     Note that we used the `toFile` built-in to write the builder and
     the stylesheet “inline” in the Nix expression. The path of the
@@ -830,13 +898,13 @@ For instance, `derivation` is also available as `builtins.derivation`.
 
   - `builtins.tryEval` *e*  
     Try to shallowly evaluate *e*. Return a set containing the
-    attributes `success` (`true` if *e* evaluated successfully, `false`
-    if an error was thrown) and `value`, equalling *e* if successful and
-    `false` otherwise. Note that this doesn't evaluate *e* deeply, so
-    ` let e = { x = throw ""; }; in (builtins.tryEval e).success
-                     ` will be `true`. Using ` builtins.deepSeq
-                     ` one can get the expected result: `let e = { x = throw "";
-                    }; in (builtins.tryEval (builtins.deepSeq e e)).success` will be
+    attributes `success` (`true` if *e* evaluated successfully,
+    `false` if an error was thrown) and `value`, equalling *e* if
+    successful and `false` otherwise. Note that this doesn't evaluate
+    *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval
+    e).success ` will be `true`. Using ` builtins.deepSeq ` one can
+    get the expected result: `let e = { x = throw ""; }; in
+    (builtins.tryEval (builtins.deepSeq e e)).success` will be
     `false`.
 
   - `builtins.typeOf` *e*  
diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/expressions/derivations.md
index 0e5656b5b..d26a33b7f 100644
--- a/doc/manual/src/expressions/derivations.md
+++ b/doc/manual/src/expressions/derivations.md
@@ -57,23 +57,34 @@ the attributes of which specify the inputs of the build.
     and it doesn’t need the documentation at build time. Thus, the
     library package could specify:
     
-        outputs = [ "lib" "headers" "doc" ];
+    ```nix
+    outputs = [ "lib" "headers" "doc" ];
+    ```
     
     This will cause Nix to pass environment variables `lib`, `headers`
     and `doc` to the builder containing the intended store paths of each
     output. The builder would typically do something like
     
-        ./configure --libdir=$lib/lib --includedir=$headers/include --docdir=$doc/share/doc
+    ```bash
+    ./configure \
+      --libdir=$lib/lib \
+      --includedir=$headers/include \
+      --docdir=$doc/share/doc
+    ```
     
     for an Autoconf-style package. You can refer to each output of a
     derivation by selecting it as an attribute, e.g.
     
-        buildInputs = [ pkg.lib pkg.headers ];
+    ```nix
+    buildInputs = [ pkg.lib pkg.headers ];
+    ```
     
     The first element of `outputs` determines the *default output*.
     Thus, you could also write
     
-        buildInputs = [ pkg pkg.headers ];
+    ```nix
+    buildInputs = [ pkg pkg.headers ];
+    ```
     
     since `pkg` is equivalent to `pkg.lib`.
 
diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md
index e3432b577..2a1306e32 100644
--- a/doc/manual/src/expressions/expression-syntax.md
+++ b/doc/manual/src/expressions/expression-syntax.md
@@ -2,17 +2,19 @@
 
 Here is a Nix expression for GNU Hello:
 
-    { stdenv, fetchurl, perl }: ①
-    
-    stdenv.mkDerivation { ②
-      name = "hello-2.1.1"; ③
-      builder = ./builder.sh; ④
-      src = fetchurl { ⑤
-        url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-        sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-      };
-      inherit perl; ⑥
-    }
+```nix
+{ stdenv, fetchurl, perl }: ①
+
+stdenv.mkDerivation { ②
+  name = "hello-2.1.1"; ③
+  builder = ./builder.sh; ④
+  src = fetchurl { ⑤
+    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+    sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+  };
+  inherit perl; ⑥
+}
+```
 
 This file is actually already in the Nix Packages collection in
 `pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to
@@ -31,31 +33,27 @@ elements (referenced from the figure by number):
     etc. `fetchurl` is a function that downloads files. `perl` is the
     Perl interpreter.
     
-    Nix functions generally have the form `{ x, y, ...,
-                    z }: e` where `x`, `y`, etc. are the names of the expected
-    arguments, and where *e* is the body of the function. So here, the
-    entire remainder of the file is the body of the function; when given
-    the required arguments, the body should describe how to build an
-    instance of the Hello package.
+    Nix functions generally have the form `{ x, y, ..., z }: e` where
+    `x`, `y`, etc. are the names of the expected arguments, and where
+    *e* is the body of the function. So here, the entire remainder of
+    the file is the body of the function; when given the required
+    arguments, the body should describe how to build an instance of
+    the Hello package.
 
 2.  So we have to build a package. Building something from other stuff
     is called a *derivation* in Nix (as opposed to sources, which are
     built by humans instead of computers). We perform a derivation by
-    calling `stdenv.mkDerivation`. `mkDerivation` is a function provided
-    by `stdenv` that builds a package from a set of *attributes*. A set
-    is just a list of key/value pairs where each key is a string and
-    each value is an arbitrary Nix expression. They take the general
-    form `{
-                    name1 =
-                    expr1; ...
-                    nameN =
-                    exprN; }`.
+    calling `stdenv.mkDerivation`. `mkDerivation` is a function
+    provided by `stdenv` that builds a package from a set of
+    *attributes*. A set is just a list of key/value pairs where each
+    key is a string and each value is an arbitrary Nix
+    expression. They take the general form `{ name1 = expr1; ...
+    nameN = exprN; }`.
 
-3.  The attribute `name` specifies the symbolic name and version of the
-    package. Nix doesn't really care about these things, but they are
-    used by for instance `nix-env
-                    -q` to show a “human-readable” name for packages. This attribute is
-    required by `mkDerivation`.
+3.  The attribute `name` specifies the symbolic name and version of
+    the package. Nix doesn't really care about these things, but they
+    are used by for instance `nix-env -q` to show a “human-readable”
+    name for packages. This attribute is required by `mkDerivation`.
 
 4.  The attribute `builder` specifies the builder. This attribute can
     sometimes be omitted, in which case `mkDerivation` will fill in a
@@ -83,8 +81,10 @@ elements (referenced from the figure by number):
     `perl` function argument to the builder. All attributes in the set
     are actually passed as environment variables to the builder, so
     declaring an attribute
-    
-        perl = perl;
+
+    ```nix
+    perl = perl;
+    ```
     
     will do the trick: it binds an attribute `perl` to the function
     argument which also happens to be called `perl`. However, it looks a
diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md
index 43275dbf7..cf26b5f82 100644
--- a/doc/manual/src/expressions/generic-builder.md
+++ b/doc/manual/src/expressions/generic-builder.md
@@ -3,12 +3,14 @@
 Recall that the [build script for GNU Hello](build-script.md) looked
 something like this:
 
-    PATH=$perl/bin:$PATH
-    tar xvfz $src
-    cd hello-*
-    ./configure --prefix=$out
-    make
-    make install
+```bash
+PATH=$perl/bin:$PATH
+tar xvfz $src
+cd hello-*
+./configure --prefix=$out
+make
+make install
+```
 
 The builders for almost all Unix packages look like this — set up some
 environment variables, unpack the sources, configure, build, and
@@ -16,11 +18,13 @@ install. For this reason the standard environment provides some Bash
 functions that automate the build process. Here is what a builder using
 the generic build facilities looks like:
 
-    buildInputs="$perl" ①
-    
-    source $stdenv/setup ②
-    
-    genericBuild ③
+```bash
+buildInputs="$perl" ①
+
+source $stdenv/setup ②
+
+genericBuild ③
+```
 
 Here is what each line means:
 
@@ -45,15 +49,17 @@ Here is what each line means:
 Discerning readers will note that the `buildInputs` could just as well
 have been set in the Nix expression, like this:
 
-``` 
+```nix
   buildInputs = [ perl ];
 ```
 
 The `perl` attribute can then be removed, and the builder becomes even
 shorter:
 
-    source $stdenv/setup
-    genericBuild
+```bash
+source $stdenv/setup
+genericBuild
+```
 
 In fact, `mkDerivation` provides a default builder that looks exactly
 like that, so it is actually possible to omit the builder for Hello
diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/expressions/language-constructs.md
index 8e267a799..cb0169239 100644
--- a/doc/manual/src/expressions/language-constructs.md
+++ b/doc/manual/src/expressions/language-constructs.md
@@ -5,10 +5,12 @@
 Recursive sets are just normal sets, but the attributes can refer to
 each other. For example,
 
-    rec {
-      x = y;
-      y = 123;
-    }.x
+```nix
+rec {
+  x = y;
+  y = 123;
+}.x
+```
 
 evaluates to `123`. Note that without `rec` the binding `x = y;` would
 refer to the variable `y` in the surrounding scope, if one exists, and
@@ -19,10 +21,12 @@ recursive set, they are.
 Recursive sets of course introduce the danger of infinite recursion. For
 example, the expression
 
-    rec {
-      x = y;
-      y = x;
-    }.x
+```nix
+rec {
+  x = y;
+  y = x;
+}.x
+```
 
 will crash with an `infinite recursion encountered` error message.
 
@@ -31,10 +35,12 @@ will crash with an `infinite recursion encountered` error message.
 A let-expression allows you to define local variables for an expression.
 For instance,
 
-    let
-      x = "foo";
-      y = "bar";
-    in x + y
+```nix
+let
+  x = "foo";
+  y = "bar";
+in x + y
+```
 
 evaluates to `"foobar"`.
 
@@ -45,38 +51,42 @@ copy variables from the surrounding lexical scope (e.g., when you want
 to propagate attributes). This can be shortened using the `inherit`
 keyword. For instance,
 
-    let x = 123; in
-    { inherit x;
-      y = 456;
-    }
+```nix
+let x = 123; in
+{ inherit x;
+  y = 456;
+}
+```
 
 is equivalent to
 
-    let x = 123; in
-    { x = x;
-      y = 456;
-    }
+```nix
+let x = 123; in
+{ x = x;
+  y = 456;
+}
+```
 
 and both evaluate to `{ x = 123; y = 456; }`. (Note that this works
 because `x` is added to the lexical scope by the `let` construct.) It is
 also possible to inherit attributes from another set. For instance, in
 this fragment from `all-packages.nix`,
 
-``` 
-  graphviz = (import ../tools/graphics/graphviz) {
-    inherit fetchurl stdenv libpng libjpeg expat x11 yacc;
-    inherit (xlibs) libXaw;
-  };
+```nix
+graphviz = (import ../tools/graphics/graphviz) {
+  inherit fetchurl stdenv libpng libjpeg expat x11 yacc;
+  inherit (xlibs) libXaw;
+};
 
-  xlibs = {
-    libX11 = ...;
-    libXaw = ...;
-    ...
-  }
-
-  libpng = ...;
-  libjpg = ...;
+xlibs = {
+  libX11 = ...;
+  libXaw = ...;
   ...
+}
+
+libpng = ...;
+libjpg = ...;
+...
 ```
 
 the set used in the function call to the function defined in
@@ -86,17 +96,21 @@ surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw`
 
 Summarizing the fragment
 
-    ...
-    inherit x y z;
-    inherit (src-set) a b c;
-    ...
+```nix
+...
+inherit x y z;
+inherit (src-set) a b c;
+...
+```
 
 is equivalent to
 
-    ...
-    x = x; y = y; z = z;
-    a = src-set.a; b = src-set.b; c = src-set.c;
-    ...
+```nix
+...
+x = x; y = y; z = z;
+a = src-set.a; b = src-set.b; c = src-set.c;
+...
+```
 
 when used while defining local variables in a let-expression or while
 defining a set.
@@ -105,7 +119,9 @@ defining a set.
 
 Functions have the following form:
 
-    pattern: body
+```nix
+pattern: body
+```
 
 The pattern specifies what the argument of the function must look like,
 and binds variables in the body to (parts of) the argument. There are
@@ -114,42 +130,51 @@ three kinds of patterns:
   - If a pattern is a single identifier, then the function matches any
     argument. Example:
     
-        let negate = x: !x;
-            concat = x: y: x + y;
-        in if negate true then concat "foo" "bar" else ""
+    ```nix
+    let negate = x: !x;
+        concat = x: y: x + y;
+    in if negate true then concat "foo" "bar" else ""
+    ```
     
     Note that `concat` is a function that takes one argument and returns
     a function that takes another argument. This allows partial
     parameterisation (i.e., only filling some of the arguments of a
     function); e.g.,
     
-        map (concat "foo") [ "bar" "bla" "abc" ]
+    ```nix
+    map (concat "foo") [ "bar" "bla" "abc" ]
+    ```
     
-    evaluates to `[ "foobar" "foobla"
-            "fooabc" ]`.
+    evaluates to `[ "foobar" "foobla" "fooabc" ]`.
 
   - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a
     set containing the listed attributes, and binds the values of those
     attributes to variables in the function body. For example, the
     function
     
-        { x, y, z }: z + y + x
+    ```nix
+    { x, y, z }: z + y + x
+    ```
     
     can only be called with a set containing exactly the attributes `x`,
     `y` and `z`. No other attributes are allowed. If you want to allow
     additional arguments, you can use an ellipsis (`...`):
     
-        { x, y, z, ... }: z + y + x
+    ```nix
+    { x, y, z, ... }: z + y + x
+    ```
     
     This works on any set that contains at least the three named
     attributes.
     
-    It is possible to provide *default values* for attributes, in which
-    case they are allowed to be missing. A default value is specified by
-    writing `name ?
-            e`, where *e* is an arbitrary expression. For example,
+    It is possible to provide *default values* for attributes, in
+    which case they are allowed to be missing. A default value is
+    specified by writing `name ?  e`, where *e* is an arbitrary
+    expression. For example,
     
-        { x, y ? "foo", z ? "bar" }: z + y + x
+    ```nix
+    { x, y ? "foo", z ? "bar" }: z + y + x
+    ```
     
     specifies a function that only requires an attribute named `x`, but
     optionally accepts `y` and `z`.
@@ -157,14 +182,14 @@ three kinds of patterns:
   - An `@`-pattern provides a means of referring to the whole value
     being matched:
     
-    ``` 
-     args@{ x, y, z, ... }: z + y + x + args.a
+    ```nix
+    args@{ x, y, z, ... }: z + y + x + args.a
     ```
     
     but can also be written as:
     
-    ``` 
-     { x, y, z, ... } @ args: z + y + x + args.a
+    ```nix
+    { x, y, z, ... } @ args: z + y + x + args.a
     ```
     
     Here `args` is bound to the entire argument, which is further
@@ -182,24 +207,30 @@ three kinds of patterns:
     > 
     > For instance
     > 
-    >     let
-    >       function = args@{ a ? 23, ... }: args;
-    >     in
-    >      function {}
+    > ```nix
+    > let
+    >   function = args@{ a ? 23, ... }: args;
+    > in
+    >   function {}
+    > ````
     > 
     > will evaluate to an empty attribute set.
 
 Note that functions do not have names. If you want to give them a name,
 you can bind them to an attribute, e.g.,
 
-    let concat = { x, y }: x + y;
-    in concat { x = "foo"; y = "bar"; }
+```nix
+let concat = { x, y }: x + y;
+in concat { x = "foo"; y = "bar"; }
+```
 
 ## Conditionals
 
 Conditionals look like this:
 
-    if e1 then e2 else e3
+```nix
+if e1 then e2 else e3
+```
 
 where *e1* is an expression that should evaluate to a Boolean value
 (`true` or `false`).
@@ -209,7 +240,9 @@ where *e1* is an expression that should evaluate to a Boolean value
 Assertions are generally used to check that certain requirements on or
 between features and dependencies hold. They look like this:
 
-    assert e1; e2
+```nix
+assert e1; e2
+```
 
 where *e1* is an expression that should evaluate to a Boolean value. If
 it evaluates to `true`, *e2* is returned; otherwise expression
@@ -218,29 +251,31 @@ evaluation is aborted and a backtrace is printed.
 Here is a Nix expression for the Subversion package that shows how
 assertions can be used:.
 
-    { localServer ? false
-    , httpServer ? false
-    , sslSupport ? false
-    , pythonBindings ? false
-    , javaSwigBindings ? false
-    , javahlBindings ? false
-    , stdenv, fetchurl
-    , openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null
-    }:
-    
-    assert localServer -> db4 != null; ①
-    assert httpServer -> httpd != null && httpd.expat == expat; ②
-    assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③
-    assert pythonBindings -> swig != null && swig.pythonSupport;
-    assert javaSwigBindings -> swig != null && swig.javaSupport;
-    assert javahlBindings -> j2sdk != null;
-    
-    stdenv.mkDerivation {
-      name = "subversion-1.1.1";
-      ...
-      openssl = if sslSupport then openssl else null; ④
-      ...
-    }
+```nix
+{ localServer ? false
+, httpServer ? false
+, sslSupport ? false
+, pythonBindings ? false
+, javaSwigBindings ? false
+, javahlBindings ? false
+, stdenv, fetchurl
+, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null
+}:
+
+assert localServer -> db4 != null; ①
+assert httpServer -> httpd != null && httpd.expat == expat; ②
+assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③
+assert pythonBindings -> swig != null && swig.pythonSupport;
+assert javaSwigBindings -> swig != null && swig.javaSupport;
+assert javahlBindings -> j2sdk != null;
+
+stdenv.mkDerivation {
+  name = "subversion-1.1.1";
+  ...
+  openssl = if sslSupport then openssl else null; ④
+  ...
+}
+```
 
 The points of interest are:
 
@@ -273,19 +308,25 @@ The points of interest are:
 
 A *with-expression*,
 
-    with e1; e2
+```nix
+with e1; e2
+```
 
 introduces the set *e1* into the lexical scope of the expression *e2*.
 For instance,
 
-    let as = { x = "foo"; y = "bar"; };
-    in with as; x + y
+```nix
+let as = { x = "foo"; y = "bar"; };
+in with as; x + y
+```
 
 evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes
 of `as` to the lexical scope in the expression `x + y`. The most common
 use of `with` is in conjunction with the `import` function. E.g.,
 
-    with (import ./definitions.nix); ...
+```nix
+with (import ./definitions.nix); ...
+```
 
 makes all attributes defined in the file `definitions.nix` available as
 if they were defined locally in a `let`-expression.
@@ -293,14 +334,17 @@ if they were defined locally in a `let`-expression.
 The bindings introduced by `with` do not shadow bindings introduced by
 other means, e.g.
 
-    let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...
+```nix
+let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...
+```
 
 establishes the same scope as
 
-    let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...
+```nix
+let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...
+```
 
 ## Comments
 
 Comments can be single-line, started with a `#` character, or
-inline/multi-line, enclosed within `/*
-... */`.
+inline/multi-line, enclosed within `/* ... */`.
diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md
index eca2cab51..ce31029cc 100644
--- a/doc/manual/src/expressions/language-values.md
+++ b/doc/manual/src/expressions/language-values.md
@@ -19,24 +19,30 @@ Nix has the following basic data types:
     into a string (meaning that it must be a string, a path, or a
     derivation). For instance, rather than writing
     
-        "--with-freetype2-library=" + freetype + "/lib"
+    ```nix
+    "--with-freetype2-library=" + freetype + "/lib"
+    ```
     
     (where `freetype` is a derivation), you can instead write the more
     natural
     
-        "--with-freetype2-library=${freetype}/lib"
+    ```nix
+    "--with-freetype2-library=${freetype}/lib"
+    ```
     
     The latter is automatically translated to the former. A more
     complicated example (from the Nix expression for
     [Qt](http://www.trolltech.com/products/qt)):
     
-        configureFlags = "
-          -system-zlib -system-libpng -system-libjpeg
-          ${if openglSupport then "-dlopen-opengl
-            -L${mesa}/lib -I${mesa}/include
-            -L${libXmu}/lib -I${libXmu}/include" else ""}
-          ${if threadSupport then "-thread" else "-no-thread"}
-        ";
+    ```nix
+    configureFlags = "
+      -system-zlib -system-libpng -system-libjpeg
+      ${if openglSupport then "-dlopen-opengl
+        -L${mesa}/lib -I${mesa}/include
+        -L${libXmu}/lib -I${libXmu}/include" else ""}
+      ${if threadSupport then "-thread" else "-no-thread"}
+    ";
+    ```
     
     Note that Nix expressions and strings can be arbitrarily nested; in
     this case the outer string contains various antiquotations that
@@ -46,11 +52,13 @@ Nix has the following basic data types:
     The second way to write string literals is as an *indented string*,
     which is enclosed between pairs of *double single-quotes*, like so:
     
-        ''
-          This is the first line.
-          This is the second line.
-            This is the third line.
-        ''
+    ```nix
+    ''
+      This is the first line.
+      This is the second line.
+        This is the third line.
+    ''
+    ```
     
     This kind of string literal intelligently strips indentation from
     the start of each line. To be precise, it strips from each line a
@@ -60,7 +68,9 @@ Nix has the following basic data types:
     line is indented four spaces. Thus, two spaces are stripped from
     each line, so the resulting string is
     
-        "This is the first line.\nThis is the second line.\n  This is the third line.\n"
+    ```nix
+    "This is the first line.\nThis is the second line.\n  This is the third line.\n"
+    ```
     
     Note that the whitespace and newline following the opening `''` is
     ignored if there is no non-whitespace text on the initial line.
@@ -82,17 +92,19 @@ Nix has the following basic data types:
     configuration files because `''` is much less common than `"`.
     Example:
     
-        stdenv.mkDerivation {
-          ...
-          postInstall =
-            ''
-              mkdir $out/bin $out/etc
-              cp foo $out/bin
-              echo "Hello World" > $out/etc/foo.conf
-              ${if enableBar then "cp bar $out/bin" else ""}
-            '';
-          ...
-        }
+    ```nix
+    stdenv.mkDerivation {
+      ...
+      postInstall =
+        ''
+          mkdir $out/bin $out/etc
+          cp foo $out/bin
+          echo "Hello World" > $out/etc/foo.conf
+          ${if enableBar then "cp bar $out/bin" else ""}
+        '';
+      ...
+    }
+    ```
     
     Finally, as a convenience, *URIs* as defined in appendix B of
     [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as
@@ -136,13 +148,17 @@ Nix has the following basic data types:
 Lists are formed by enclosing a whitespace-separated list of values
 between square brackets. For example,
 
-    [ 123 ./foo.nix "abc" (f { x = y; }) ]
+```nix
+[ 123 ./foo.nix "abc" (f { x = y; }) ]
+```
 
 defines a list of four elements, the last being the result of a call to
 the function `f`. Note that function calls have to be enclosed in
 parentheses. If they had been omitted, e.g.,
 
-    [ 123 ./foo.nix "abc" f { x = y; } ]
+```nix
+[ 123 ./foo.nix "abc" f { x = y; } ]
+```
 
 the result would be a list of five elements, the fourth one being a
 function and the fifth being a set.
@@ -159,10 +175,12 @@ Sets are just a list of name/value pairs (called *attributes*) enclosed
 in curly brackets, where each value is an arbitrary expression
 terminated by a semicolon. For example:
 
-    { x = 123;
-      text = "Hello";
-      y = f { bla = 456; };
-    }
+```nix
+{ x = 123;
+  text = "Hello";
+  y = f { bla = 456; };
+}
+```
 
 This defines a set with attributes named `x`, `text`, `y`. The order of
 the attributes is irrelevant. An attribute name may only occur once.
@@ -170,24 +188,32 @@ the attributes is irrelevant. An attribute name may only occur once.
 Attributes can be selected from a set using the `.` operator. For
 instance,
 
-    { a = "Foo"; b = "Bar"; }.a
+```nix
+{ a = "Foo"; b = "Bar"; }.a
+```
 
 evaluates to `"Foo"`. It is possible to provide a default value in an
 attribute selection using the `or` keyword. For example,
 
-    { a = "Foo"; b = "Bar"; }.c or "Xyzzy"
+```nix
+{ a = "Foo"; b = "Bar"; }.c or "Xyzzy"
+```
 
 will evaluate to `"Xyzzy"` because there is no `c` attribute in the set.
 
 You can use arbitrary double-quoted strings as attribute names:
 
-    { "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}"
+```nix
+{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}"
+```
 
 This will evaluate to `123` (Assuming `bar` is antiquotable). In the
 case where an attribute name is just a single antiquotation, the quotes
 can be dropped:
 
-    { foo = 123; }.${bar} or 456 
+```nix
+{ foo = 123; }.${bar} or 456
+```
 
 This will evaluate to `123` if `bar` evaluates to `"foo"` when coerced
 to a string and `456` otherwise (again assuming `bar` is antiquotable).
@@ -196,7 +222,9 @@ In the special case where an attribute name inside of a set declaration
 evaluates to `null` (which is normally an error, as `null` is not
 antiquotable), that attribute is simply not added to the set:
 
-    { ${if foo then "bar" else null} = true; }
+```nix
+{ ${if foo then "bar" else null} = true; }
+```
 
 This will evaluate to `{}` if `foo` evaluates to `false`.
 
@@ -205,9 +233,11 @@ itself a function or a set with a `__functor` attribute whose value is
 callable) can be applied as if it were a function, with the set itself
 passed in first , e.g.,
 
-    let add = { __functor = self: x: x + self.x; };
-        inc = add // { x = 1; };
-    in inc 1
+```nix
+let add = { __functor = self: x: x + self.x; };
+    inc = add // { x = 1; };
+in inc 1
+```
 
 evaluates to `2`. This can be used to attach metadata to a function
 without the caller needing to treat it specially, or to implement a form
diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md
index ca422acea..6f730a936 100644
--- a/doc/manual/src/expressions/simple-building-testing.md
+++ b/doc/manual/src/expressions/simple-building-testing.md
@@ -6,18 +6,20 @@ yet. The best way to test the package is by using the command
 `nix-build`, which builds a Nix expression and creates a symlink named
 `result` in the current directory:
 
-    $ nix-build -A hello
-    building path `/nix/store/632d2b22514d...-hello-2.1.1'
-    hello-2.1.1/
-    hello-2.1.1/intl/
-    hello-2.1.1/intl/ChangeLog
-    ...
-    
-    $ ls -l result
-    lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1
-    
-    $ ./result/bin/hello
-    Hello, world!
+```console
+$ nix-build -A hello
+building path `/nix/store/632d2b22514d...-hello-2.1.1'
+hello-2.1.1/
+hello-2.1.1/intl/
+hello-2.1.1/intl/ChangeLog
+...
+
+$ ls -l result
+lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1
+
+$ ./result/bin/hello
+Hello, world!
+```
 
 The `-A` option selects the `hello` attribute. This is faster than
 using the symbolic package name specified by the `name` attribute
@@ -50,8 +52,10 @@ simultaneously, and they try to build the same derivation, the first Nix
 instance that gets there will perform the build, while the others block
 (or perform other derivations if available) until the build finishes:
 
-    $ nix-build -A hello
-    waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'
+```console
+$ nix-build -A hello
+waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'
+```
 
 So it is always safe to run multiple instances of Nix in parallel (which
 isn’t the case with, say, `make`).
diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index f8375822d..1aa4e6b5f 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -3,45 +3,63 @@
 This section provides some notes on how to hack on Nix. To get the
 latest version of Nix from GitHub:
 
-    $ git clone https://github.com/NixOS/nix.git
-    $ cd nix
+```console
+$ git clone https://github.com/NixOS/nix.git
+$ cd nix
+```
 
 To build Nix for the current operating system/architecture use
 
-    $ nix-build
+```console
+$ nix-build
+```
 
 or if you have a flakes-enabled nix:
 
-    $ nix build
+```console
+$ nix build
+```
 
 This will build `defaultPackage` attribute defined in the `flake.nix`
 file. To build for other platforms add one of the following suffixes to
 it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e.
 
-    $ nix-build -A defaultPackage.x86_64-linux
+```console
+$ nix-build -A defaultPackage.x86_64-linux
+```
 
 To build all dependencies and start a shell in which all environment
 variables are set up so that those dependencies can be found:
 
-    $ nix-shell
+```console
+$ nix-shell
+```
 
 To build Nix itself in this shell:
 
-    [nix-shell]$ ./bootstrap.sh
-    [nix-shell]$ ./configure $configureFlags
-    [nix-shell]$ make -j $NIX_BUILD_CORES
+```console
+[nix-shell]$ ./bootstrap.sh
+[nix-shell]$ ./configure $configureFlags
+[nix-shell]$ make -j $NIX_BUILD_CORES
+```
 
 To install it in `$(pwd)/inst` and test it:
 
-    [nix-shell]$ make install
-    [nix-shell]$ make installcheck
-    [nix-shell]$ ./inst/bin/nix --version
-    nix (Nix) 2.4
+```console
+[nix-shell]$ make install
+[nix-shell]$ make installcheck
+[nix-shell]$ ./inst/bin/nix --version
+nix (Nix) 2.4
+```
 
-If you have a flakes-enabled nix you can replace:
+If you have a flakes-enabled Nix you can replace:
 
-    $ nix-shell
+```console
+$ nix-shell
+```
 
 by:
 
-    $ nix develop
+```console
+$ nix develop
+```
diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md
index 35dbd5541..d21a51a82 100644
--- a/doc/manual/src/installation/building-source.md
+++ b/doc/manual/src/installation/building-source.md
@@ -3,16 +3,20 @@
 After unpacking or checking out the Nix sources, issue the following
 commands:
 
-    $ ./configure options...
-    $ make
-    $ make install
+```console
+$ ./configure options...
+$ make
+$ make install
+```
 
 Nix requires GNU Make so you may need to invoke `gmake` instead.
 
 When building from the Git repository, these should be preceded by the
 command:
 
-    $ ./bootstrap.sh
+```console
+$ ./bootstrap.sh
+```
 
 The installation path can be specified by passing the `--prefix=prefix`
 to `configure`. The default installation directory is `/usr/local`. You
diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md
index 6e78245c9..4a49897e4 100644
--- a/doc/manual/src/installation/env-variables.md
+++ b/doc/manual/src/installation/env-variables.md
@@ -10,7 +10,9 @@ environment variables is to include the file
 `prefix/etc/profile.d/nix.sh` in your `~/.profile` (or similar), like
 this:
 
-    source prefix/etc/profile.d/nix.sh
+```bash
+source prefix/etc/profile.d/nix.sh
+```
 
 # `NIX_SSL_CERT_FILE`
 
@@ -23,13 +25,17 @@ and use its own certificate bundle.
 
 Set the environment variable and install Nix
 
-    $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-    $ sh <(curl -L https://nixos.org/nix/install)
+```console
+$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+$ sh <(curl -L https://nixos.org/nix/install)
+```
 
 In the shell profile and rc files (for example, `/etc/bashrc`,
 `/etc/zshrc`), add the following line:
 
-    export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+```bash
+export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+```
 
 > **Note**
 > 
@@ -41,8 +47,10 @@ In the shell profile and rc files (for example, `/etc/bashrc`,
 On macOS you must specify the environment variable for the Nix daemon
 service, then restart it:
 
-    $ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt
-    $ sudo launchctl kickstart -k system/org.nixos.nix-daemon
+```console
+$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt
+$ sudo launchctl kickstart -k system/org.nixos.nix-daemon
+```
 
 ## Proxy Environment Variables
 
diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index a1dd993c4..8b8d1d738 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -3,8 +3,8 @@
 If you are using Linux or macOS versions up to 10.14 (Mojave), the
 easiest way to install Nix is to run the following command:
 
-``` 
-  $ sh <(curl -L https://nixos.org/nix/install)
+```console
+$ sh <(curl -L https://nixos.org/nix/install)
 ```
 
 If you're using macOS 10.15 (Catalina) or newer, consult [the macOS
@@ -18,8 +18,8 @@ installation is highly recommended.
 
 To explicitly select a single-user installation on your system:
 
-``` 
-  sh <(curl -L https://nixos.org/nix/install) --no-daemon
+```console
+$ sh <(curl -L https://nixos.org/nix/install) --no-daemon
 ```
 
 This will perform a single-user installation of Nix, meaning that `/nix`
@@ -28,8 +28,10 @@ account, *not* as root. The script will invoke `sudo` to create `/nix`
 if it doesn’t already exist. If you don’t have `sudo`, you should
 manually create `/nix` first as root, e.g.:
 
-    $ mkdir /nix
-    $ chown alice /nix
+```console
+$ mkdir /nix
+$ chown alice /nix
+```
 
 The install script will modify the first writable file from amongst
 `.bash_profile`, `.bash_login` and `.profile` to source
@@ -39,7 +41,9 @@ the install script to disable this behaviour.
 
 You can uninstall Nix simply by running:
 
-    $ rm -rf /nix
+```console
+$ rm -rf /nix
+```
 
 # Multi User Installation
 
@@ -53,7 +57,9 @@ service for the Nix daemon.
 You can instruct the installer to perform a multi-user installation on
 your system:
 
-    sh <(curl -L https://nixos.org/nix/install) --daemon
+```console
+$ sh <(curl -L https://nixos.org/nix/install) --daemon
+```
 
 The multi-user installation of Nix will create build users between the
 user IDs 30001 and 30032, and a group with the group ID 30000. You
@@ -72,18 +78,20 @@ extension. The installer will also create `/etc/profile.d/nix.sh`.
 
 You can uninstall Nix with the following commands:
 
-    sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
-    
-    # If you are on Linux with systemd, you will need to run:
-    sudo systemctl stop nix-daemon.socket
-    sudo systemctl stop nix-daemon.service
-    sudo systemctl disable nix-daemon.socket
-    sudo systemctl disable nix-daemon.service
-    sudo systemctl daemon-reload
-    
-    # If you are on macOS, you will need to run:
-    sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-    sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+```console
+sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+
+# If you are on Linux with systemd, you will need to run:
+sudo systemctl stop nix-daemon.socket
+sudo systemctl stop nix-daemon.service
+sudo systemctl disable nix-daemon.socket
+sudo systemctl disable nix-daemon.service
+sudo systemctl daemon-reload
+
+# If you are on macOS, you will need to run:
+sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+```
 
 There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
 and `/etc/zshrc` which you may remove.
@@ -110,7 +118,9 @@ chip](https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview
 your drive will still be encrypted at rest (in which case "unencrypted"
 is a bit of a misnomer). To use this approach, just install Nix with:
 
-    $ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume
+```console
+$ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume
+```
 
 If you don't like the sound of this, you'll want to weigh the other
 approaches and tradeoffs detailed in this section.
@@ -184,7 +194,9 @@ there are a few things to weigh:
 If you are comfortable navigating these tradeoffs, you can encrypt the
 volume with something along the lines of:
 
-    alice$ diskutil apfs enableFileVault /nix -user disk
+```console
+alice$ diskutil apfs enableFileVault /nix -user disk
+```
 
 ## Symlink the Nix store to a custom location
 
@@ -221,11 +233,15 @@ as a helpful reference if you run into trouble.
     `apfs.util` to trigger creation (not deletion) of new entries
     without a reboot:
     
-        alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+    ```console
+    alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+    ```
 
 3.  Create the new APFS volume with diskutil:
     
-        alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
+    ```console
+    alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
+    ```
 
 4.  Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't
     already have other entries, it should look something like:
@@ -248,8 +264,8 @@ since 1.11.16, at `https://releases.nixos.org/nix/nix-version/install`.
 These install scripts can be used the same as the main NixOS.org
 installation script:
 
-``` 
-  sh <(curl -L https://nixos.org/nix/install)
+```console
+$ sh <(curl -L https://nixos.org/nix/install)
 ```
 
 In the same directory of the install script are sha256 sums, and gpg
@@ -263,10 +279,12 @@ dependencies. (This is what the install script at
 it somewhere (e.g. in `/tmp`), and then run the script named `install`
 inside the binary tarball:
 
-    alice$ cd /tmp
-    alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-    alice$ cd nix-1.8-x86_64-darwin
-    alice$ ./install
+```console
+alice$ cd /tmp
+alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
+alice$ cd nix-1.8-x86_64-darwin
+alice$ ./install
+```
 
 If you need to edit the multi-user installation script to use different
 group ID or a different user ID range, modify the variables set in the
diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/src/installation/multi-user.md
index de159b603..6920591c4 100644
--- a/doc/manual/src/installation/multi-user.md
+++ b/doc/manual/src/installation/multi-user.md
@@ -28,10 +28,12 @@ group should have no other members. The build users should not be
 members of any other group. On Linux, you can create the group and users
 as follows:
 
-    $ groupadd -r nixbld
-    $ for n in $(seq 1 10); do useradd -c "Nix build user $n" \
-        -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \
-        nixbld$n; done
+```console
+$ groupadd -r nixbld
+$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \
+    -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \
+    nixbld$n; done
+```
 
 This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if you
@@ -42,7 +44,9 @@ expect to do many builds at the same time.
 The [Nix daemon](../command-ref/nix-daemon.md) should be started as
 follows (as `root`):
 
-    $ nix-daemon
+```console
+$ nix-daemon
+```
 
 You’ll want to put that line somewhere in your system’s boot scripts.
 
@@ -50,7 +54,9 @@ To let unprivileged users use the daemon, they should set the
 [`NIX_REMOTE` environment variable](../command-ref/env-common.md) to
 `daemon`. So you should put a line like
 
-    export NIX_REMOTE=daemon
+```console
+export NIX_REMOTE=daemon
+```
 
 into the users’ login scripts.
 
@@ -61,8 +67,10 @@ permissions on the directory `/nix/var/nix/daemon-socket`. For instance,
 if you want to restrict the use of Nix to the members of a group called
 `nix-users`, do
 
-    $ chgrp nix-users /nix/var/nix/daemon-socket
-    $ chmod ug=rwx,o= /nix/var/nix/daemon-socket
+```console
+$ chgrp nix-users /nix/var/nix/daemon-socket
+$ chmod ug=rwx,o= /nix/var/nix/daemon-socket
+```
 
 This way, users who are not in the `nix-users` group cannot connect to
 the Unix domain socket `/nix/var/nix/daemon-socket/socket`, so they
diff --git a/doc/manual/src/installation/obtaining-source.md b/doc/manual/src/installation/obtaining-source.md
index 2937130cf..0a906e390 100644
--- a/doc/manual/src/installation/obtaining-source.md
+++ b/doc/manual/src/installation/obtaining-source.md
@@ -10,7 +10,9 @@ Alternatively, the most recent sources of Nix can be obtained from its
 following command will check out the latest revision into a directory
 called `nix`:
 
-    $ git clone https://github.com/NixOS/nix
+```console
+$ git clone https://github.com/NixOS/nix
+```
 
 Likewise, specific releases can be obtained from the
 [tags](https://github.com/NixOS/nix/tags) of the repository.
diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md
index b54b0d02d..f01fe7b38 100644
--- a/doc/manual/src/introduction.md
+++ b/doc/manual/src/introduction.md
@@ -75,21 +75,27 @@ And since packages aren’t overwritten, the old versions are still
 there after an upgrade.  This means that you can _roll back_ to the
 old version:
 
-    $ nix-env --upgrade some-packages
-    $ nix-env --rollback
+```console
+$ nix-env --upgrade some-packages
+$ nix-env --rollback
+```
 
 ## Garbage collection
 
 When you uninstall a package like this…
 
-    $ nix-env --uninstall firefox
+```console
+$ nix-env --uninstall firefox
+```
 
 the package isn’t deleted from the system right away (after all, you
 might want to do a rollback, or it might be in the profiles of other
 users).  Instead, unused packages can be deleted safely by running the
 _garbage collector_:
 
-    $ nix-collect-garbage
+```console
+$ nix-collect-garbage
+```
 
 This deletes all packages that aren’t in use by any user profile or by
 a currently running program.
@@ -115,7 +121,9 @@ each other in the Nix store.
 Nix expressions generally describe how to build a package from
 source, so an installation action like
 
-    $ nix-env --install firefox
+```console
+$ nix-env --install firefox
+```
 
 _could_ cause quite a bit of build activity, as not only Firefox but
 also all its dependencies (all the way up to the C library and the
@@ -149,16 +157,20 @@ For example, the following command gets all dependencies of the
 Pan newsreader, as described by [its
 Nix expression](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix):
 
-    $ nix-shell '<nixpkgs>' -A pan
+```console
+$ nix-shell '<nixpkgs>' -A pan
+```
 
 You’re then dropped into a shell where you can edit, build and test
 the package:
 
-    [nix-shell]$ tar xf $src
-    [nix-shell]$ cd pan-*
-    [nix-shell]$ ./configure
-    [nix-shell]$ make
-    [nix-shell]$ ./pan/gui/pan
+```console
+[nix-shell]$ tar xf $src
+[nix-shell]$ cd pan-*
+[nix-shell]$ ./configure
+[nix-shell]$ make
+[nix-shell]$ ./pan/gui/pan
+```
 
 ## Portability
 
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
index 17b5cc9c2..9702a29eb 100644
--- a/doc/manual/src/package-management/basic-package-mgmt.md
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -31,8 +31,10 @@ automatically added to your list of “subscribed” channels when you
 install Nix. If this is not the case for some reason, you can add it
 as follows:
 
-    $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-    $ nix-channel --update
+```console
+$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+$ nix-channel --update
+```
 
 > **Note**
 > 
@@ -44,14 +46,16 @@ as follows:
 
 You can view the set of available packages in Nixpkgs:
 
-    $ nix-env -qa
-    aterm-2.2
-    bash-3.0
-    binutils-2.15
-    bison-1.875d
-    blackdown-1.4.2
-    bzip2-1.0.2
-    …
+```console
+$ nix-env -qa
+aterm-2.2
+bash-3.0
+binutils-2.15
+bison-1.875d
+blackdown-1.4.2
+bzip2-1.0.2
+…
+```
 
 The flag `-q` specifies a query operation, and `-a` means that you want
 to show the “available” (i.e., installable) packages, as opposed to the
@@ -59,31 +63,39 @@ installed packages. If you downloaded Nixpkgs yourself, or if you
 checked it out from GitHub, then you need to pass the path to your
 Nixpkgs tree using the `-f` flag:
 
-    $ nix-env -qaf /path/to/nixpkgs
+```console
+$ nix-env -qaf /path/to/nixpkgs
+```
 
 where */path/to/nixpkgs* is where you’ve unpacked or checked out
 Nixpkgs.
 
 You can select specific packages by name:
 
-    $ nix-env -qa firefox
-    firefox-34.0.5
-    firefox-with-plugins-34.0.5
+```console
+$ nix-env -qa firefox
+firefox-34.0.5
+firefox-with-plugins-34.0.5
+```
 
 and using regular expressions:
 
-    $ nix-env -qa 'firefox.*'
+```console
+$ nix-env -qa 'firefox.*'
+```
 
 It is also possible to see the *status* of available packages, i.e.,
 whether they are installed into the user environment and/or present in
 the system:
 
-    $ nix-env -qas
-    …
-    -PS bash-3.0
-    --S binutils-2.15
-    IPS bison-1.875d
-    …
+```console
+$ nix-env -qas
+…
+-PS bash-3.0
+--S binutils-2.15
+IPS bison-1.875d
+…
+```
 
 The first character (`I`) indicates whether the package is installed in
 your current user environment. The second (`P`) indicates whether it is
@@ -96,7 +108,9 @@ Nix knows that it can fetch a pre-built package from somewhere
 
 You can install a package using `nix-env -i`. For instance,
 
-    $ nix-env -i subversion
+```console
+$ nix-env -i subversion
+```
 
 will install the package called `subversion` (which is, of course, the
 [Subversion version management system](http://subversion.tigris.org/)).
@@ -121,12 +135,16 @@ will install the package called `subversion` (which is, of course, the
 
 Naturally, packages can also be uninstalled:
 
-    $ nix-env -e subversion
+```console
+$ nix-env -e subversion
+```
 
 Upgrading to a new version is just as easy. If you have a new release of
 Nix Packages, you can do:
 
-    $ nix-env -u subversion
+```console
+$ nix-env -u subversion
+```
 
 This will *only* upgrade Subversion if there is a “newer” version in the
 new set of Nix expressions, as defined by some pretty arbitrary rules
@@ -137,14 +155,18 @@ whatever version is in the Nix expressions, use `-i` instead of `-u`;
 
 You can also upgrade all packages for which there are newer versions:
 
-    $ nix-env -u
+```console
+$ nix-env -u
+```
 
 Sometimes it’s useful to be able to ask what `nix-env` would do, without
 actually doing it. For instance, to find out what packages would be
 upgraded by `nix-env -u`, you can do
 
-    $ nix-env -u --dry-run
-    (dry run; not doing anything)
-    upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
-    upgrading `graphviz-1.10' to `graphviz-1.12'
-    upgrading `coreutils-5.0' to `coreutils-5.2.1'
+```console
+$ nix-env -u --dry-run
+(dry run; not doing anything)
+upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
+upgrading `graphviz-1.10' to `graphviz-1.12'
+upgrading `coreutils-5.0' to `coreutils-5.2.1'
+```
diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md
index 44f0da238..bdc5038fc 100644
--- a/doc/manual/src/package-management/binary-cache-substituter.md
+++ b/doc/manual/src/package-management/binary-cache-substituter.md
@@ -8,16 +8,22 @@ usually uses to fetch pre-built binaries from <https://cache.nixos.org>.
 The daemon that handles binary cache requests via HTTP, `nix-serve`, is
 not part of the Nix distribution, but you can install it from Nixpkgs:
 
-    $ nix-env -i nix-serve
+```console
+$ nix-env -i nix-serve
+```
 
 You can then start the server, listening for HTTP connections on
 whatever port you like:
 
-    $ nix-serve -p 8080
+```console
+$ nix-serve -p 8080
+```
 
 To check whether it works, try the following on the client:
 
-    $ curl http://avalon:8080/nix-cache-info
+```console
+$ curl http://avalon:8080/nix-cache-info
+```
 
 which should print something like:
 
@@ -28,7 +34,9 @@ which should print something like:
 On the client side, you can tell Nix to use your binary cache using
 `--option extra-binary-caches`, e.g.:
 
-    $ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
+```console
+$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
+```
 
 The option `extra-binary-caches` tells Nix to use this binary cache in
 addition to your default caches, such as <https://cache.nixos.org>.
diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md
index c239998d9..93c8b41a6 100644
--- a/doc/manual/src/package-management/channels.md
+++ b/doc/manual/src/package-management/channels.md
@@ -15,7 +15,9 @@ To see the list of official NixOS channels, visit
 
 You can “subscribe” to a channel using `nix-channel --add`, e.g.,
 
-    $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+```console
+$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+```
 
 subscribes you to a channel that always contains that latest version of
 the Nix Packages collection. (Subscribing really just means that the URL
@@ -24,11 +26,15 @@ calls to `nix-channel
 --update`.) You can “unsubscribe” using `nix-channel
 --remove`:
 
-    $ nix-channel --remove nixpkgs
+```console
+$ nix-channel --remove nixpkgs
+```
 
 To obtain the latest Nix expressions available in a channel, do
 
-    $ nix-channel --update
+```console
+$ nix-channel --update
+```
 
 This downloads and unpacks the Nix expressions in every channel
 (downloaded from `url/nixexprs.tar.bz2`). It also makes the union of
@@ -36,7 +42,9 @@ each channel’s Nix expressions available by default to `nix-env`
 operations (via the symlink `~/.nix-defexpr/channels`). Consequently,
 you can then say
 
-    $ nix-env -u
+```console
+$ nix-env -u
+```
 
 to upgrade all packages in your profile to the latest versions available
 in the subscribed channels.
diff --git a/doc/manual/src/package-management/garbage-collection.md b/doc/manual/src/package-management/garbage-collection.md
index 4c8799dfe..fecb30fd6 100644
--- a/doc/manual/src/package-management/garbage-collection.md
+++ b/doc/manual/src/package-management/garbage-collection.md
@@ -18,23 +18,31 @@ be done if you are certain that you will not need to roll back.
 
 To delete all old (non-current) generations of your current profile:
 
-    $ nix-env --delete-generations old
+```console
+$ nix-env --delete-generations old
+```
 
 Instead of `old` you can also specify a list of generations, e.g.,
 
-    $ nix-env --delete-generations 10 11 14
+```console
+$ nix-env --delete-generations 10 11 14
+```
 
 To delete all generations older than a specified number of days (except
 the current generation), use the `d` suffix. For example,
 
-    $ nix-env --delete-generations 14d
+```console
+$ nix-env --delete-generations 14d
+```
 
 deletes all generations older than two weeks.
 
 After removing appropriate old generations you can run the garbage
 collector as follows:
 
-    $ nix-store --gc
+```console
+$ nix-store --gc
+```
 
 The behaviour of the gargage collector is affected by the
 `keep-derivations` (default: true) and `keep-outputs` (default: false)
@@ -47,7 +55,9 @@ sense to keep outputs to ensure that rebuild times are quick.) If you
 are feeling uncertain, you can also first view what files would be
 deleted:
 
-    $ nix-store --gc --print-dead
+```console
+$ nix-store --gc --print-dead
+```
 
 Likewise, the option `--print-live` will show the paths that *won’t* be
 deleted.
@@ -56,6 +66,8 @@ There is also a convenient little utility `nix-collect-garbage`, which
 when invoked with the `-d` (`--delete-old`) switch deletes all old
 generations of all profiles in `/nix/var/nix/profiles`. So
 
-    $ nix-collect-garbage -d
+```console
+$ nix-collect-garbage -d
+```
 
 is a quick and easy way to clean up your system.
diff --git a/doc/manual/src/package-management/garbage-collector-roots.md b/doc/manual/src/package-management/garbage-collector-roots.md
index 6f4d48e80..30c5b7f8d 100644
--- a/doc/manual/src/package-management/garbage-collector-roots.md
+++ b/doc/manual/src/package-management/garbage-collector-roots.md
@@ -5,7 +5,9 @@ are symlinks in the directory `prefix/nix/var/nix/gcroots`. For
 instance, the following command makes the path
 `/nix/store/d718ef...-foo` a root of the collector:
 
-    $ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
+```console
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar
+```
 
 That is, after this command, the garbage collector will not remove
 `/nix/store/d718ef...-foo` or any of its dependencies.
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
index f3786072d..fbbfb7320 100644
--- a/doc/manual/src/package-management/profiles.md
+++ b/doc/manual/src/package-management/profiles.md
@@ -22,7 +22,9 @@ store looks like:
 
 Of course, you wouldn’t want to type
 
-    $ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn
+```console
+$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn
+```
 
 every time you want to run Subversion. Of course we could set up the
 `PATH` environment variable to include the `bin` directory of every
@@ -36,7 +38,9 @@ environment `/nix/store/0c1p5z4kda11...-user-env` contains a symlink to
 just Subversion 1.1.2 (arrows in the figure indicate symlinks). This
 would be what we would obtain if we had done
 
-    $ nix-env -i subversion
+```console
+$ nix-env -i subversion
+```
 
 on a set of Nix expressions that contained Subversion 1.1.2.
 
@@ -49,7 +53,9 @@ since every time you perform a `nix-env` operation, a new user
 environment is generated based on the current one. For instance,
 generation 43 was created from generation 42 when we did
 
-    $ nix-env -i subversion firefox
+```console
+$ nix-env -i subversion firefox
+```
 
 on a set of Nix expressions that contained Firefox and a new version of
 Subversion.
@@ -57,11 +63,13 @@ Subversion.
 Generations are grouped together into *profiles* so that different users
 don’t interfere with each other if they don’t want to. For example:
 
-    $ ls -l /nix/var/nix/profiles/
-    ...
-    lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
-    lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
-    lrwxrwxrwx  1 eelco ... default -> default-43-link
+```console
+$ ls -l /nix/var/nix/profiles/
+...
+lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
+lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
+lrwxrwxrwx  1 eelco ... default -> default-43-link
+```
 
 This shows a profile called `default`. The file `default` itself is
 actually a symlink that points to the current generation. When we do a
@@ -75,18 +83,24 @@ store.)
 
 If you find that you want to undo a `nix-env` operation, you can just do
 
-    $ nix-env --rollback
+```console
+$ nix-env --rollback
+```
 
 which will just make the current generation link point at the previous
 link. E.g., `default` would be made to point at `default-42-link`. You
 can also switch to a specific generation:
 
-    $ nix-env --switch-generation 43
+```console
+$ nix-env --switch-generation 43
+```
 
 which in this example would roll forward to generation 43 again. You can
 also see all available generations:
 
-    $ nix-env --list-generations
+```console
+$ nix-env --list-generations
+```
 
 You generally wouldn’t have `/nix/var/nix/profiles/some-profile/bin` in
 your `PATH`. Rather, there is a symlink `~/.nix-profile` that points to
@@ -96,9 +110,11 @@ initialisation script `/nix/etc/profile.d/nix.sh` does). This makes it
 easier to switch to a different profile. You can do that using the
 command `nix-env --switch-profile`:
 
-    $ nix-env --switch-profile /nix/var/nix/profiles/my-profile
-    
-    $ nix-env --switch-profile /nix/var/nix/profiles/default
+```console
+$ nix-env --switch-profile /nix/var/nix/profiles/my-profile
+
+$ nix-env --switch-profile /nix/var/nix/profiles/default
+```
 
 These commands switch to the `my-profile` and default profile,
 respectively. If the profile doesn’t exist, it will be created
@@ -110,6 +126,8 @@ All `nix-env` operations work on the profile pointed to by
 `~/.nix-profile`, but you can override this using the `--profile` option
 (abbreviation `-p`):
 
-    $ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
+```console
+$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
+```
 
 This will *not* change the `~/.nix-profile` symlink.
diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md
index 2824c1a9b..a4f4d561f 100644
--- a/doc/manual/src/package-management/s3-substituter.md
+++ b/doc/manual/src/package-management/s3-substituter.md
@@ -51,25 +51,27 @@ cache's documentation.
 
 Your bucket will need the following bucket policy:
 
-    {
-        "Id": "DirectReads",
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Sid": "AllowDirectReads",
-                "Action": [
-                    "s3:GetObject",
-                    "s3:GetBucketLocation"
-                ],
-                "Effect": "Allow",
-                "Resource": [
-                    "arn:aws:s3:::example-nix-cache",
-                    "arn:aws:s3:::example-nix-cache/*"
-                ],
-                "Principal": "*"
-            }
-        ]
-    }
+```json
+{
+    "Id": "DirectReads",
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AllowDirectReads",
+            "Action": [
+                "s3:GetObject",
+                "s3:GetBucketLocation"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::example-nix-cache",
+                "arn:aws:s3:::example-nix-cache/*"
+            ],
+            "Principal": "*"
+        }
+    ]
+}
+```
 
 ## Authenticated Reads to your S3 binary cache
 
@@ -101,35 +103,43 @@ for authenticating requests to Amazon S3.
 
 Your account will need the following IAM policy to upload to the cache:
 
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
     {
-      "Version": "2012-10-17",
-      "Statement": [
-        {
-          "Sid": "UploadToCache",
-          "Effect": "Allow",
-          "Action": [
-            "s3:AbortMultipartUpload",
-            "s3:GetBucketLocation",
-            "s3:GetObject",
-            "s3:ListBucket",
-            "s3:ListBucketMultipartUploads",
-            "s3:ListMultipartUploadParts",
-            "s3:PutObject"
-          ],
-          "Resource": [
-            "arn:aws:s3:::example-nix-cache",
-            "arn:aws:s3:::example-nix-cache/*"
-          ]
-        }
+      "Sid": "UploadToCache",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:ListMultipartUploadParts",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::example-nix-cache",
+        "arn:aws:s3:::example-nix-cache/*"
       ]
     }
+  ]
+}
+```
 
 ## Examples
 
 To upload with a specific credential profile for Amazon S3:
 
-    nix copy --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2' nixpkgs.hello
+```console
+$ nix copy nixpkgs.hello \
+  --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2'
+```
 
 To upload to an S3-compatible binary cache:
 
-    nix copy --to 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' nixpkgs.hello
+```console
+$ nix copy nixpkgs.hello --to \
+  's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
+```
diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md
index 482844c7c..6e5e258bc 100644
--- a/doc/manual/src/package-management/ssh-substituter.md
+++ b/doc/manual/src/package-management/ssh-substituter.md
@@ -5,7 +5,9 @@ Nix store via SSH. For example, the following installs Firefox,
 automatically fetching any store paths in Firefox’s closure if they are
 available on the server `avalon`:
 
-    $ nix-env -i firefox --substituters ssh://alice@avalon
+```console
+$ nix-env -i firefox --substituters ssh://alice@avalon
+```
 
 This works similar to the binary cache substituter that Nix usually
 uses, only using SSH instead of HTTP: if a store path `P` is needed, Nix
@@ -22,11 +24,17 @@ building from source.
 You can also copy the closure of some store path, without installing it
 into your profile, e.g.
 
-    $ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon
+```console
+$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters
+ssh://alice@avalon
+```
 
 This is essentially equivalent to doing
 
-    $ nix-copy-closure --from alice@avalon /nix/store/m85bxg…-firefox-34.0.5
+```console
+$ nix-copy-closure --from alice@avalon
+/nix/store/m85bxg…-firefox-34.0.5
+```
 
 You can use SSH’s *forced command* feature to set up a restricted user
 account for SSH substituter access, allowing read-only access to the
@@ -45,8 +53,10 @@ to `sshd_config` to restrict the user `nix-ssh`:
 On NixOS, you can accomplish the same by adding the following to your
 `configuration.nix`:
 
-    nix.sshServe.enable = true;
-    nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+```nix
+nix.sshServe.enable = true;
+nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+```
 
 where the latter line lists the public keys of users that are allowed to
 connect.
diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 3c519217b..71205923b 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -6,7 +6,9 @@ to subsequent chapters.
 
 1. Install single-user Nix by running the following:
 
-        $ bash <(curl -L https://nixos.org/nix/install)
+   ```console
+   $ bash <(curl -L https://nixos.org/nix/install)
+   ```
 
    This will install Nix in `/nix`. The install script will create
    `/nix` using `sudo`, so make sure you have sufficient rights.  (For
@@ -16,52 +18,66 @@ to subsequent chapters.
 1. See what installable packages are currently available in the
    channel:
 
-        $ nix-env -qa
-        docbook-xml-4.3
-        docbook-xml-4.5
-        firefox-33.0.2
-        hello-2.9
-        libxslt-1.1.28
-        …
+   ```console
+   $ nix-env -qa
+   docbook-xml-4.3
+   docbook-xml-4.5
+   firefox-33.0.2
+   hello-2.9
+   libxslt-1.1.28
+   …
+   ```
 
 1. Install some packages from the channel:
 
-        $ nix-env -i hello
+   ```console
+   $ nix-env -i hello
+   ```
 
    This should download pre-built packages; it should not build them
    locally (if it does, something went wrong).
 
 1. Test that they work:
 
-        $ which hello
-        /home/eelco/.nix-profile/bin/hello
-        $ hello
-        Hello, world!
+   ```console
+   $ which hello
+   /home/eelco/.nix-profile/bin/hello
+   $ hello
+   Hello, world!
+   ```
 
 1. Uninstall a package:
 
-        $ nix-env -e hello
+   ```console
+   $ nix-env -e hello
+   ```
 
 1. You can also test a package without installing it:
 
-        $ nix-shell -p hello
+   ```console
+   $ nix-shell -p hello
+   ```
 
    This builds or downloads GNU Hello and its dependencies, then drops
    you into a Bash shell where the `hello` command is present, all
    without affecting your normal environment:
 
-        [nix-shell:~]$ hello
-        Hello, world!
+   ```console
+   [nix-shell:~]$ hello
+   Hello, world!
 
-        [nix-shell:~]$ exit
+   [nix-shell:~]$ exit
 
-        $ hello
-        hello: command not found
+   $ hello
+   hello: command not found
+   ```
 
 1. To keep up-to-date with the channel, do:
 
-        $ nix-channel --update nixpkgs
-        $ nix-env -u '*'
+   ```console
+   $ nix-channel --update nixpkgs
+   $ nix-env -u '*'
+   ```
 
    The latter command will upgrade each installed package for which
    there is a “newer” version (as determined by comparing the version
@@ -70,10 +86,14 @@ to subsequent chapters.
 1. If you're unhappy with the result of a `nix-env` action (e.g., an
    upgraded package turned out not to work properly), you can go back:
 
-        $ nix-env --rollback
+   ```console
+   $ nix-env --rollback
+   ```
 
 1. You should periodically run the Nix garbage collector to get rid of
    unused packages, since uninstalls or upgrades don't actually delete
    them:
 
-        $ nix-collect-garbage -d
+   ```console
+   $ nix-collect-garbage -d
+   ```
diff --git a/flake.nix b/flake.nix
index f31dc4039..8f5a3055c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -467,6 +467,7 @@
               configureFlags+=" --prefix=$prefix"
               PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
               PATH=$prefix/bin:$PATH
+              export MANPATH=/home/eelco/Dev/nix/inst/share/man:$MANPATH
               unset PYTHONPATH
             '';
         });
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index d884a131e..ed5791b14 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -49,7 +49,8 @@ struct NarAccessor : public FSAccessor
             : acc(acc), source(source)
         { }
 
-        void createMember(const Path & path, NarMember member) {
+        void createMember(const Path & path, NarMember member)
+        {
             size_t level = std::count(path.begin(), path.end(), '/');
             while (parents.size() > level) parents.pop();
 

From 2ae9ac23698276c3e045d9b21f5c6ffb40f38324 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Jul 2020 16:02:37 +0200
Subject: [PATCH 053/882] console -> shell

---
 doc/manual/src/command-ref/nix-shell.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 27826717b..fc42a202a 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -110,7 +110,7 @@ The following common options are supported:
 To build the dependencies of the package Pan, and start an interactive
 shell in which to build it:
 
-```shell
+```console
 $ nix-shell '<nixpkgs>' -A pan
 [nix-shell]$ unpackPhase
 [nix-shell]$ cd pan-*
@@ -122,7 +122,7 @@ $ nix-shell '<nixpkgs>' -A pan
 To clear the environment first, and do some additional automatic
 initialisation of the interactive shell:
 
-```shell
+```console
 $ nix-shell '<nixpkgs>' -A pan --pure \
     --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
 ```
@@ -131,13 +131,13 @@ Nix expressions can also be given on the command line using the `-E` and
 `-p` flags. For instance, the following starts a shell containing the
 packages `sqlite` and `libX11`:
 
-```shell
+```console
 $ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
 ```
 
 A shorter way to do the same is:
 
-```shell
+```console
 $ nix-shell -p sqlite xorg.libX11
 [nix-shell]$ echo $NIX_LDFLAGS
 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
@@ -147,7 +147,7 @@ Note that `-p` accepts multiple full nix expressions that are valid in
 the `buildInputs = [ ... ]` shown above, not only package names. So the
 following is also legal:
 
-```shell
+```console
 $ nix-shell -p sqlite 'git.override { withManual = false; }'
 ```
 
@@ -156,7 +156,7 @@ it by passing `-I` or setting `NIX_PATH`. For example, the following
 gives you a shell containing the Pan package from a specific revision of
 Nixpkgs:
 
-```shell
+```console
 $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version

From 2f2ae993dc6d35e9c0e66e893e5d615116d42917 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 4 Aug 2020 19:02:05 +0000
Subject: [PATCH 054/882] WIP systematize more of the worker protocol

This refactor should *not* change the wire protocol.
---
 src/libstore/build.cc            |   4 +-
 src/libstore/daemon.cc           |  34 ++++-----
 src/libstore/derivations.cc      |   4 +-
 src/libstore/export-import.cc    |   4 +-
 src/libstore/legacy-ssh-store.cc |  14 ++--
 src/libstore/remote-store.cc     | 116 +++++++++++++------------------
 src/libstore/store-api.cc        |  16 ++---
 src/libstore/worker-protocol.hh  |  59 ++++++++++------
 src/nix-store/nix-store.cc       |  16 ++---
 9 files changed, 132 insertions(+), 135 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1f40dc42a..dc636c33f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1864,11 +1864,11 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    writeStorePaths(worker.store, hook->sink, inputPaths);
+    write(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
-    writeStorePaths(worker.store, hook->sink, missingPaths);
+    write(worker.store, hook->sink, missingPaths);
 
     hook->sink = FdSink();
     hook->toHook.writeSide = -1;
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 9f7be6e1a..f92d384e5 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -256,11 +256,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = readStorePaths<StorePathSet>(*store, from);
+        auto paths = read(*store, from, Proxy<StorePathSet> {});
         logger->startWork();
         auto res = store->queryValidPaths(paths);
         logger->stopWork();
-        writeStorePaths(*store, to, res);
+        write(*store, to, res);
         break;
     }
 
@@ -276,11 +276,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = readStorePaths<StorePathSet>(*store, from);
+        auto paths = read(*store, from, Proxy<StorePathSet> {});
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        writeStorePaths(*store, to, res);
+        write(*store, to, res);
         break;
     }
 
@@ -309,7 +309,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        writeStorePaths(*store, to, paths);
+        write(*store, to, paths);
         break;
     }
 
@@ -397,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopAddTextToStore: {
         string suffix = readString(from);
         string s = readString(from);
-        auto refs = readStorePaths<StorePathSet>(*store, from);
+        auto refs = read(*store, from, Proxy<StorePathSet> {});
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -518,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
+        options.pathsToDelete = read(*store, from, Proxy<StorePathSet> {});
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -587,7 +587,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            writeStorePaths(*store, to, i->second.references);
+            write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -598,11 +598,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = readStorePaths<StorePathSet>(*store, from);
+            auto paths = read(*store, from, Proxy<StorePathSet> {});
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = readStorePathCAMap(*store, from);
+            pathsMap = read(*store, from, Proxy<StorePathCAMap> {});
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -610,7 +610,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            writeStorePaths(*store, to, i.second.references);
+            write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        writeStorePaths(*store, to, paths);
+        write(*store, to, paths);
         break;
     }
 
@@ -639,7 +639,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 to << 1;
             to << (info->deriver ? store->printStorePath(*info->deriver) : "")
                << info->narHash->to_string(Base16, false);
-            writeStorePaths(*store, to, info->references);
+            write(*store, to, info->references);
             to << info->registrationTime << info->narSize;
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
@@ -699,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
         info.narHash = Hash(readString(from), htSHA256);
-        info.references = readStorePaths<StorePathSet>(*store, from);
+        info.references = read(*store, from, Proxy<StorePathSet> {});
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = parseContentAddressOpt(readString(from));
@@ -799,9 +799,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        writeStorePaths(*store, to, willBuild);
-        writeStorePaths(*store, to, willSubstitute);
-        writeStorePaths(*store, to, unknown);
+        write(*store, to, willBuild);
+        write(*store, to, willSubstitute);
+        write(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 7d0a5abeb..5972b5ad2 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -477,7 +477,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
+    drv.inputSrcs = read(store, in, Proxy<StorePathSet> {});
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -505,7 +505,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             out << "" << "";
         }
     }
-    writeStorePaths(store, out, drv.inputSrcs);
+    write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index a0fc22264..c20c56156 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    writeStorePaths(*this, teeSink, info->references);
+    write(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        info.references = readStorePaths<StorePathSet>(*this, source);
+        info.references = read(*this, source, Proxy<StorePathSet> {});
 
         auto deriver = readString(source);
         if (deriver != "")
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5d7566121..412e1950b 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -107,7 +107,7 @@ struct LegacySSHStore : public Store
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = readStorePaths<StorePathSet>(*this, conn->from);
+            info->references = read(*this, conn->from, Proxy<StorePathSet> {});
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -139,7 +139,7 @@ struct LegacySSHStore : public Store
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash->to_string(Base16, false);
-            writeStorePaths(*this, conn->to, info.references);
+            write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -168,7 +168,7 @@ struct LegacySSHStore : public Store
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            writeStorePaths(*this, conn->to, info.references);
+            write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -251,10 +251,10 @@ struct LegacySSHStore : public Store
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        writeStorePaths(*this, conn->to, paths);
+        write(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
+        for (auto & i : read(*this, conn->from, Proxy<StorePathSet> {}))
             out.insert(i);
     }
 
@@ -267,10 +267,10 @@ struct LegacySSHStore : public Store
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        writeStorePaths(*this, conn->to, paths);
+        write(*this, conn->to, paths);
         conn->to.flush();
 
-        return readStorePaths<StorePathSet>(*this, conn->from);
+        return read(*this, conn->from, Proxy<StorePathSet> {});
     }
 
     void connect() override
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1200ab200..de50b3e2e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -23,66 +23,44 @@
 namespace nix {
 
 
-template<> StorePathSet readStorePaths(const Store & store, Source & from)
-{
-    StorePathSet paths;
-    for (auto & i : readStrings<Strings>(from))
-        paths.insert(store.parseStorePath(i));
-    return paths;
-}
-
-
-void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
+void write(const Store & store, Sink & out, const StorePathSet & paths)
 {
     out << paths.size();
     for (auto & i : paths)
         out << store.printStorePath(i);
 }
 
+
+std::string read(const Store & store, Source & from, Proxy<std::string> _)
+{
+    return readString(from);
+}
+
+void write(const Store & store, Sink & out, const std::string & str)
+{
+    out << str;
+}
+
+
 StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
 {
-	auto path = readString(from);
-	return store.parseStorePath(path);
+    return store.parseStorePath(readString(from));
 }
 
-StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
-{
-    StorePathCAMap paths;
-    auto count = readNum<size_t>(from);
-    while (count--)
-        paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
-    return paths;
-}
-
-void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
-{
-    out << paths.size();
-    for (auto & i : paths) {
-        out << store.printStorePath(i.first);
-        out << renderContentAddress(i.second);
-    }
-}
-
-std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
-{
-    std::map<string, StorePath> pathMap;
-    auto rawInput = readStrings<Strings>(from);
-    if (rawInput.size() % 2)
-        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
-    auto curInput = rawInput.begin();
-    while (curInput != rawInput.end()) {
-        auto thisKey = *curInput++;
-        auto thisValue = *curInput++;
-        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
-    }
-    return pathMap;
-}
-
-
 void write(const Store & store, Sink & out, const StorePath & storePath)
 {
-	auto path = store.printStorePath(storePath);
-	out << path;
+    out << store.printStorePath(storePath);
+}
+
+
+ContentAddress read(const Store & store, Source & from, Proxy<ContentAddress> _)
+{
+    return parseContentAddress(readString(from));
+}
+
+void write(const Store & store, Sink & out, const ContentAddress & ca)
+{
+    out << renderContentAddress(ca);
 }
 
 
@@ -319,9 +297,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        writeStorePaths(*this, conn->to, paths);
+        write(*this, conn->to, paths);
         conn.processStderr();
-        return readStorePaths<StorePathSet>(*this, conn->from);
+        return read(*this, conn->from, Proxy<StorePathSet> {});
     }
 }
 
@@ -331,7 +309,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return readStorePaths<StorePathSet>(*this, conn->from);
+    return read(*this, conn->from, Proxy<StorePathSet> {});
 }
 
 
@@ -348,9 +326,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        writeStorePaths(*this, conn->to, paths);
+        write(*this, conn->to, paths);
         conn.processStderr();
-        return readStorePaths<StorePathSet>(*this, conn->from);
+        return read(*this, conn->from, Proxy<StorePathSet> {});
     }
 }
 
@@ -372,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = readStorePaths<StorePathSet>(*this, conn->from);
+            info.references = read(*this, conn->from, Proxy<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -385,9 +363,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            writeStorePaths(*this, conn->to, paths);
+            write(*this, conn->to, paths);
         } else
-            writeStorePathCAMap(*this, conn->to, pathsMap);
+            write(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -395,7 +373,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = readStorePaths<StorePathSet>(*this, conn->from);
+            info.references = read(*this, conn->from, Proxy<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -428,7 +406,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
             auto deriver = readString(conn->from);
             if (deriver != "") info->deriver = parseStorePath(deriver);
             info->narHash = Hash(readString(conn->from), htSHA256);
-            info->references = readStorePaths<StorePathSet>(*this, conn->from);
+            info->references = read(*this, conn->from, Proxy<StorePathSet> {});
             conn->from >> info->registrationTime >> info->narSize;
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
                 conn->from >> info->ultimate;
@@ -447,7 +425,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
+    for (auto & i : read(*this, conn->from, Proxy<StorePathSet> {}))
         referrers.insert(i);
 }
 
@@ -457,7 +435,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return readStorePaths<StorePathSet>(*this, conn->from);
+    return read(*this, conn->from, Proxy<StorePathSet> {});
 }
 
 
@@ -469,7 +447,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return readStorePaths<StorePathSet>(*this, conn->from);
+    return read(*this, conn->from, Proxy<StorePathSet> {});
 }
 
 
@@ -508,7 +486,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            writeStorePaths(*this, sink, info.references);
+            write(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -518,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from);
+        auto importedPaths = read(*this, conn->from, Proxy<StorePathSet> {});
         assert(importedPaths.size() <= 1);
     }
 
@@ -527,7 +505,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash->to_string(Base16, false);
-        writeStorePaths(*this, conn->to, info.references);
+        write(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -660,7 +638,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
 
     auto conn(getConnection());
     conn->to << wopAddTextToStore << name << s;
-    writeStorePaths(*this, conn->to, references);
+    write(*this, conn->to, references);
 
     conn.processStderr();
     return parseStorePath(readString(conn->from));
@@ -762,7 +740,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << wopCollectGarbage << options.action;
-    writeStorePaths(*this, conn->to, options.pathsToDelete);
+    write(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -824,9 +802,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
             ss.push_back(p.to_string(*this));
         conn->to << ss;
         conn.processStderr();
-        willBuild = readStorePaths<StorePathSet>(*this, conn->from);
-        willSubstitute = readStorePaths<StorePathSet>(*this, conn->from);
-        unknown = readStorePaths<StorePathSet>(*this, conn->from);
+        willBuild = read(*this, conn->from, Proxy<StorePathSet> {});
+        willSubstitute = read(*this, conn->from, Proxy<StorePathSet> {});
+        unknown = read(*this, conn->from, Proxy<StorePathSet> {});
         conn->from >> downloadSize >> narSize;
         return;
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 4c68709ef..e894d2b85 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -362,14 +362,14 @@ bool Store::PathInfoCacheValue::isKnownNow()
 }
 
 OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path) {
-	auto resp = queryDerivationOutputMap(path);
-	OutputPathMap result;
-	for (auto & [outName, optOutPath] : resp) {
-		if (!optOutPath)
-			throw Error("output '%s' has no store path mapped to it", outName);
-		result.insert_or_assign(outName, *optOutPath);
-	}
-	return result;
+    auto resp = queryDerivationOutputMap(path);
+    OutputPathMap result;
+    for (auto & [outName, optOutPath] : resp) {
+        if (!optOutPath)
+            throw Error("output '%s' has no store path mapped to it", outName);
+        result.insert_or_assign(outName, *optOutPath);
+    }
+    return result;
 }
 
 StorePathSet Store::queryDerivationOutputs(const StorePath & path)
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index ad5854c85..08eec9b48 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -66,33 +66,50 @@ typedef enum {
 class Store;
 struct Source;
 
-template<class T> T readStorePaths(const Store & store, Source & from);
-
-void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
-
 /* To guide overloading */
 template<typename T>
 struct Proxy {};
 
 template<typename T>
-std::map<std::string, T> read(const Store & store, Source & from, Proxy<std::map<std::string, T>> _)
+std::set<T> read(const Store & store, Source & from, Proxy<std::set<T>> _)
 {
-    std::map<string, T> resMap;
-    auto size = (size_t)readInt(from);
+    std::set<T> resSet;
+    auto size = readNum<size_t>(from);
     while (size--) {
-        auto thisKey = readString(from);
-        resMap.insert_or_assign(std::move(thisKey), read(store, from, Proxy<T> {}));
+        resSet.insert(read(store, from, Proxy<T> {}));
+    }
+    return resSet;
+}
+
+template<typename T>
+void write(const Store & store, Sink & out, const std::set<T> & resSet)
+{
+    out << resSet.size();
+    for (auto & key : resSet) {
+        write(store, out, key);
+    }
+}
+
+template<typename K, typename V>
+std::map<K, V> read(const Store & store, Source & from, Proxy<std::map<K, V>> _)
+{
+    std::map<K, V> resMap;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        resMap.insert_or_assign(
+            read(store, from, Proxy<K> {}),
+            read(store, from, Proxy<V> {}));
     }
     return resMap;
 }
 
-template<typename T>
-void write(const Store & store, Sink & out, const std::map<string, T> & resMap)
+template<typename K, typename V>
+void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
 {
     out << resMap.size();
-    for (auto & i : resMap) {
-        out << i.first;
-        write(store, out, i.second);
+    for (auto & [key, value] : resMap) {
+        write(store, out, key);
+        write(store, out, value);
     }
 }
 
@@ -106,26 +123,28 @@ std::optional<T> read(const Store & store, Source & from, Proxy<std::optional<T>
     case 1:
         return read(store, from, Proxy<T> {});
     default:
-        throw Error("got an invalid tag bit for std::optional: %#04x", tag);
+        throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
     }
 }
 
 template<typename T>
 void write(const Store & store, Sink & out, const std::optional<T> & optVal)
 {
-    out << (optVal ? 1 : 0);
+    out << (uint64_t) (optVal ? 1 : 0);
     if (optVal)
         write(store, out, *optVal);
 }
 
+std::string read(const Store & store, Source & from, Proxy<std::string> _);
+
+void write(const Store & store, Sink & out, const std::string & str);
+
 StorePath read(const Store & store, Source & from, Proxy<StorePath> _);
 
 void write(const Store & store, Sink & out, const StorePath & storePath);
 
-StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
+ContentAddress read(const Store & store, Source & from, Proxy<ContentAddress> _);
 
-void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
-
-void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
+void write(const Store & store, Sink & out, const ContentAddress & ca);
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 7b26970ef..a1fb921ef 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -815,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = readStorePaths<StorePathSet>(*store, in);
+                auto paths = read(*store, in, Proxy<StorePathSet> {});
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -845,19 +845,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                         }
                 }
 
-                writeStorePaths(*store, out, store->queryValidPaths(paths));
+                write(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                auto paths = readStorePaths<StorePathSet>(*store, in);
+                auto paths = read(*store, in, Proxy<StorePathSet> {});
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        writeStorePaths(*store, out, info->references);
+                        write(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -885,7 +885,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(readStorePaths<StorePathSet>(*store, in), out);
+                store->exportPaths(read(*store, in, Proxy<StorePathSet> {}), out);
                 break;
             }
 
@@ -934,9 +934,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(readStorePaths<StorePathSet>(*store, in),
+                store->computeFSClosure(read(*store, in, Proxy<StorePathSet> {}),
                     closure, false, includeOutputs);
-                writeStorePaths(*store, out, closure);
+                write(*store, out, closure);
                 break;
             }
 
@@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
                 info.narHash = Hash(readString(in), htSHA256);
-                info.references = readStorePaths<StorePathSet>(*store, in);
+                info.references = read(*store, in, Proxy<StorePathSet> {});
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = parseContentAddressOpt(readString(in));

From 1bab8a321f246c1c202268851b0e706ff5030d2e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 4 Aug 2020 21:56:42 +0000
Subject: [PATCH 055/882] Remove unneeded definition

Template instantiations will cover this case fine.
---
 src/libstore/remote-store.cc | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index de50b3e2e..5de8f95a7 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -23,14 +23,6 @@
 namespace nix {
 
 
-void write(const Store & store, Sink & out, const StorePathSet & paths)
-{
-    out << paths.size();
-    for (auto & i : paths)
-        out << store.printStorePath(i);
-}
-
-
 std::string read(const Store & store, Source & from, Proxy<std::string> _)
 {
     return readString(from);

From 45b6fdb22b05352108b6470ae16d611431c06666 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 4 Aug 2020 22:10:13 +0000
Subject: [PATCH 056/882] Remove unused functions

---
 src/libstore/remote-store.cc    | 16 ----------------
 src/libstore/worker-protocol.hh |  2 --
 2 files changed, 18 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1200ab200..dc2efefeb 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -63,22 +63,6 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
     }
 }
 
-std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
-{
-    std::map<string, StorePath> pathMap;
-    auto rawInput = readStrings<Strings>(from);
-    if (rawInput.size() % 2)
-        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
-    auto curInput = rawInput.begin();
-    while (curInput != rawInput.end()) {
-        auto thisKey = *curInput++;
-        auto thisValue = *curInput++;
-        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
-    }
-    return pathMap;
-}
-
-
 void write(const Store & store, Sink & out, const StorePath & storePath)
 {
 	auto path = store.printStorePath(storePath);
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index ad5854c85..117d3e1a4 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -126,6 +126,4 @@ StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
 
 void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
 
-void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
-
 }

From 1dfcbebc95f1c74763770bb2bd7afdebe8804845 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 4 Aug 2020 22:28:10 +0000
Subject: [PATCH 057/882] Organize and format code a bit

---
 src/libstore/remote-store.cc | 17 +++++++++--------
 src/libstore/store-api.cc    | 16 ++++++++--------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index dc2efefeb..772979066 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -31,7 +31,6 @@ template<> StorePathSet readStorePaths(const Store & store, Source & from)
     return paths;
 }
 
-
 void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
 {
     out << paths.size();
@@ -39,11 +38,6 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
         out << store.printStorePath(i);
 }
 
-StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
-{
-	auto path = readString(from);
-	return store.parseStorePath(path);
-}
 
 StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
 {
@@ -63,10 +57,17 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
     }
 }
 
+
+StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
+{
+    auto path = readString(from);
+    return store.parseStorePath(path);
+}
+
 void write(const Store & store, Sink & out, const StorePath & storePath)
 {
-	auto path = store.printStorePath(storePath);
-	out << path;
+    auto path = store.printStorePath(storePath);
+    out << path;
 }
 
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 4c68709ef..e894d2b85 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -362,14 +362,14 @@ bool Store::PathInfoCacheValue::isKnownNow()
 }
 
 OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path) {
-	auto resp = queryDerivationOutputMap(path);
-	OutputPathMap result;
-	for (auto & [outName, optOutPath] : resp) {
-		if (!optOutPath)
-			throw Error("output '%s' has no store path mapped to it", outName);
-		result.insert_or_assign(outName, *optOutPath);
-	}
-	return result;
+    auto resp = queryDerivationOutputMap(path);
+    OutputPathMap result;
+    for (auto & [outName, optOutPath] : resp) {
+        if (!optOutPath)
+            throw Error("output '%s' has no store path mapped to it", outName);
+        result.insert_or_assign(outName, *optOutPath);
+    }
+    return result;
 }
 
 StorePathSet Store::queryDerivationOutputs(const StorePath & path)

From 16c98bf57c52dee59c06e0e8911800943a468962 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 4 Aug 2020 22:36:31 +0000
Subject: [PATCH 058/882] Get rid of some unneeded temporaries

---
 src/libstore/remote-store.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 772979066..63c19a450 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -60,14 +60,12 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
 
 StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
 {
-    auto path = readString(from);
-    return store.parseStorePath(path);
+    return store.parseStorePath(readString(from));
 }
 
 void write(const Store & store, Sink & out, const StorePath & storePath)
 {
-    auto path = store.printStorePath(storePath);
-    out << path;
+    out << store.printStorePath(storePath);
 }
 
 

From a9bbfaa8515de7107457fda2a7aef87aa198788e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 5 Aug 2020 16:27:15 +0000
Subject: [PATCH 059/882] Fix --profile with multiple opaque paths

---
 src/nix/command.cc | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 04715b43a..da32819da 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -128,27 +128,25 @@ void MixProfile::updateProfile(const Buildables & buildables)
 {
     if (!profile) return;
 
-    std::optional<StorePath> result;
+    std::vector<StorePath> result;
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
             [&](BuildableOpaque bo) {
-                result = bo.path;
+                result.push_back(bo.path);
             },
             [&](BuildableFromDrv bfd) {
                 for (auto & output : bfd.outputs) {
-                    if (result)
-                        throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
-                    result = output.second;
+                    result.push_back(output.second);
                 }
             },
         }, buildable);
     }
 
-    if (!result)
-        throw Error("'--profile' requires that the arguments produce a single store path, but there are none");
+    if (result.size() != 1)
+        throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
-    updateProfile(*result);
+    updateProfile(result[0]);
 }
 
 MixDefaultProfile::MixDefaultProfile()

From f1a47a96b6ab55d0b0017df5b94b3da69c65bf21 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 5 Aug 2020 10:58:00 -0600
Subject: [PATCH 060/882] error messages for issue 2238

---
 src/build-remote/build-remote.cc | 33 +++++++++++++++++++++++++++++++-
 src/libstore/build.cc            | 13 +++++++++++--
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 3579d8fff..2414a47c1 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -103,7 +103,7 @@ static int _main(int argc, char * * argv)
             drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-             auto canBuildLocally = amWilling
+            auto canBuildLocally = amWilling
                  &&  (  neededSystem == settings.thisSystem
                      || settings.extraPlatforms.get().count(neededSystem) > 0)
                  &&  allSupportedLocally(requiredFeatures);
@@ -170,7 +170,38 @@ static int _main(int argc, char * * argv)
                     if (rightType && !canBuildLocally)
                         std::cerr << "# postpone\n";
                     else
+                    {
+                        // build the hint template.
+                        string hintstring =  "required (system, features): (%s, %s)";
+                        hintstring += "\n%s available machines:";
+                        hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+
+                        for (unsigned int i = 0; i < machines.size(); ++i) {
+                          hintstring += "\n(%s, %s, %s, %s)";
+                        }
+
+                        // add the template values.
+                        auto hint = hintformat(hintstring);
+                        hint
+                          % neededSystem
+                          % concatStringsSep<StringSet>(", ", requiredFeatures)
+                          % machines.size();
+
+                        for (auto & m : machines) {
+                          hint % concatStringsSep<vector<string>>(", ", m.systemTypes)
+                            % m.maxJobs
+                            % concatStringsSep<StringSet>(", ", m.supportedFeatures)
+                            % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
+                        }
+
+                        logError({
+                              .name = "Remote build",
+                              .description = "Failed to find a machine for remote build!",
+                              .hint = hint
+                        });
+
                         std::cerr << "# decline\n";
+                    }
                     break;
                 }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 94c398b2f..76baa1a6e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4876,8 +4876,17 @@ void Worker::run(const Goals & _topGoals)
             waitForInput();
         else {
             if (awake.empty() && 0 == settings.maxBuildJobs)
-                throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds");
+            {
+                if (getMachines().empty())
+                   throw Error("unable to start any build; either increase '--max-jobs' "
+                            "or enable remote builds."
+                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                else
+                   throw Error("unable to start any build; remote machines may not have "
+                            "all required system features."
+                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+
+            }
             assert(!awake.empty());
         }
     }

From e4eae078a52fa4209bb5a46d21dbed09dd9c54ec Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 5 Aug 2020 11:21:36 -0600
Subject: [PATCH 061/882] add derivation path to hint

---
 src/build-remote/build-remote.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 2414a47c1..723e1463e 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -172,7 +172,7 @@ static int _main(int argc, char * * argv)
                     else
                     {
                         // build the hint template.
-                        string hintstring =  "required (system, features): (%s, %s)";
+                        string hintstring =  "derivation: %s\nrequired (system, features): (%s, %s)";
                         hintstring += "\n%s available machines:";
                         hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
 
@@ -183,6 +183,7 @@ static int _main(int argc, char * * argv)
                         // add the template values.
                         auto hint = hintformat(hintstring);
                         hint
+                          % drvPath->to_string()
                           % neededSystem
                           % concatStringsSep<StringSet>(", ", requiredFeatures)
                           % machines.size();

From 31f1af0cabb65e3c5f8f4539500c6b236c387780 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 5 Aug 2020 11:26:06 -0600
Subject: [PATCH 062/882] don't crash if there's no drvPath

---
 src/build-remote/build-remote.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 723e1463e..5247cefa6 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -181,9 +181,15 @@ static int _main(int argc, char * * argv)
                         }
 
                         // add the template values.
+                        string drvstr;
+                        if (drvPath.has_value())
+                            drvstr = drvPath->to_string();
+                        else
+                            drvstr = "<unknown>";
+
                         auto hint = hintformat(hintstring);
                         hint
-                          % drvPath->to_string()
+                          % drvstr
                           % neededSystem
                           % concatStringsSep<StringSet>(", ", requiredFeatures)
                           % machines.size();

From 1d71028f4d43f8e3ade559f7187f697c7bb9d709 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Wed, 5 Aug 2020 14:42:48 -0400
Subject: [PATCH 063/882] Remove optionality in ValidPathInfo::narInfo

---
 src/libexpr/primops/fetchTree.cc    |  2 +-
 src/libfetchers/fetchers.cc         |  4 ++--
 src/libstore/build.cc               |  2 +-
 src/libstore/daemon.cc              |  4 ++--
 src/libstore/export-import.cc       |  4 ++--
 src/libstore/legacy-ssh-store.cc    |  4 ++--
 src/libstore/local-store.cc         | 14 +++++++-------
 src/libstore/nar-info-disk-cache.cc |  2 +-
 src/libstore/nar-info.cc            |  4 ++--
 src/libstore/path-info.hh           |  6 +++---
 src/libstore/remote-store.cc        |  2 +-
 src/libstore/store-api.cc           |  6 +++---
 src/nix-store/nix-store.cc          | 10 +++++-----
 src/nix/make-content-addressable.cc |  2 +-
 src/nix/profile.cc                  |  2 +-
 src/nix/verify.cc                   |  8 ++++----
 16 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index cddcf0e59..0dbf4ae1d 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             : hashFile(htSHA256, path);
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-                *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
+                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
     if (state.allowedPaths)
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 9c69fc564..eaa635595 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -130,12 +130,12 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
         tree.actualPath = store->toRealPath(tree.storePath);
 
     auto narHash = store->queryPathInfo(tree.storePath)->narHash;
-    input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
+    input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
 
     if (auto prevNarHash = getNarHash()) {
         if (narHash != *prevNarHash)
             throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-                to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
+                to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
     }
 
     if (auto prevLastModified = getLastModified()) {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4156cd678..51dd0c094 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -5037,7 +5037,7 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
+        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
         Hash nullHash(htSHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 5e568fc94..fa3bf6e2d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -289,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto hash = store->queryPathInfo(path)->narHash;
         logger->stopWork();
-        to << hash->to_string(Base16, false);
+        to << hash.to_string(Base16, false);
         break;
     }
 
@@ -638,7 +638,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
                 to << 1;
             to << (info->deriver ? store->printStorePath(*info->deriver) : "")
-               << info->narHash->to_string(Base16, false);
+               << info->narHash.to_string(Base16, false);
             writeStorePaths(*store, to, info->references);
             to << info->registrationTime << info->narSize;
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index a0fc22264..3f4b7c36f 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -38,9 +38,9 @@ void Store::exportPath(const StorePath & path, Sink & sink)
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashSink.currentHash().first;
-    if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
+    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
-            printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
+            printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
     teeSink
         << exportMagic
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index c6eeab548..ed57a188b 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
 
             if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
                 auto s = readString(conn->from);
-                info->narHash = s.empty() ? std::optional<Hash>{} : Hash::parseAnyPrefixed(s);
+                info->narHash = s.empty() ? Hash(htSHA256) : Hash::parseAnyPrefixed(s);
                 info->ca = parseContentAddressOpt(readString(conn->from));
                 info->sigs = readStrings<StringSet>(conn->from);
             }
@@ -138,7 +138,7 @@ struct LegacySSHStore : public Store
                 << cmdAddToStoreNar
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
-                << info.narHash->to_string(Base16, false);
+                << info.narHash.to_string(Base16, false);
             writeStorePaths(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 6b0548538..93a86f828 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -579,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     state.stmtRegisterValidPath.use()
         (printStorePath(info.path))
-        (info.narHash->to_string(Base16, true))
+        (info.narHash.to_string(Base16, true))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
         (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
         (info.narSize, info.narSize != 0)
@@ -679,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
     state.stmtUpdatePathInfo.use()
         (info.narSize, info.narSize != 0)
-        (info.narHash->to_string(Base16, true))
+        (info.narHash.to_string(Base16, true))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (renderContentAddress(info.ca), (bool) info.ca)
@@ -905,7 +905,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         StorePathSet paths;
 
         for (auto & i : infos) {
-            assert(i.narHash && i.narHash->type == htSHA256);
+            assert(i.narHash && i.narHash.type == htSHA256);
             if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
             else
@@ -1018,7 +1018,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
-                    printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));
+                    printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
@@ -1300,9 +1300,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 std::unique_ptr<AbstractHashSink> hashSink;
                 if (!info->ca || !info->references.count(info->path))
-                    hashSink = std::make_unique<HashSink>(info->narHash->type);
+                    hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
@@ -1311,7 +1311,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                     logError({
                         .name = "Invalid hash - path modified",
                         .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
+                        printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
                     });
                     if (repair) repairPath(i); else errors = true;
                 } else {
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 92da14e23..dd40006d7 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -232,7 +232,7 @@ public:
                     (narInfo ? narInfo->compression : "", narInfo != 0)
                     (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
                     (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
-                    (info->narHash->to_string(Base32, true))
+                    (info->narHash.to_string(Base32, true))
                     (info->narSize)
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 5812aa4ac..cadaf6267 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -89,8 +89,8 @@ std::string NarInfo::to_string(const Store & store) const
     assert(fileHash && fileHash->type == htSHA256);
     res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
-    assert(narHash && narHash->type == htSHA256);
-    res += "NarHash: " + narHash->to_string(Base32, true) + "\n";
+    assert(narHash && narHash.type == htSHA256);
+    res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 2a015ea3c..ba154dab7 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -29,7 +29,7 @@ struct ValidPathInfo
     StorePath path;
     std::optional<StorePath> deriver;
     // TODO document this
-    std::optional<Hash> narHash;
+    Hash narHash;
     StorePathSet references;
     time_t registrationTime = 0;
     uint64_t narSize = 0; // 0 = unknown
@@ -100,8 +100,8 @@ struct ValidPathInfo
 
     ValidPathInfo(const ValidPathInfo & other) = default;
 
-    ValidPathInfo(StorePath && path) : path(std::move(path)) { };
-    ValidPathInfo(const StorePath & path) : path(path) { };
+    ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash(htSHA256)) { };
+    ValidPathInfo(const StorePath & path) : path(path), narHash(Hash(htSHA256)) { };
 
     virtual ~ValidPathInfo() { }
 };
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 33d1e431b..b5e4d9fa6 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -521,7 +521,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
         conn->to << wopAddToStoreNar
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
-                 << info.narHash->to_string(Base16, false);
+                 << info.narHash.to_string(Base16, false);
         writeStorePaths(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ea666f098..81180c484 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -569,7 +569,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
         auto info = queryPathInfo(i);
 
         if (showHash) {
-            s += info->narHash->to_string(Base16, false) + "\n";
+            s += info->narHash.to_string(Base16, false) + "\n";
             s += (format("%1%\n") % info->narSize).str();
         }
 
@@ -601,7 +601,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
             auto info = queryPathInfo(storePath);
 
             jsonPath
-                .attr("narHash", info->narHash->to_string(hashBase, true))
+                .attr("narHash", info->narHash.to_string(hashBase, true))
                 .attr("narSize", info->narSize);
 
             {
@@ -919,7 +919,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
             store.printStorePath(path));
     return
         "1;" + store.printStorePath(path) + ";"
-        + narHash->to_string(Base32, true) + ";"
+        + narHash.to_string(Base32, true) + ";"
         + std::to_string(narSize) + ";"
         + concatStringsSep(",", store.printStorePathSet(references));
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 2996e36c4..a901261f4 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
-                        assert(info->narHash && info->narHash->type == htSHA256);
-                        cout << fmt("%s\n", info->narHash->to_string(Base32, true));
+                        assert(info->narHash && info->narHash.type == htSHA256);
+                        cout << fmt("%s\n", info->narHash.to_string(Base32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
                 }
@@ -723,7 +723,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = store->queryPathInfo(path);
-        HashSink sink(info->narHash->type);
+        HashSink sink(info->narHash.type);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
@@ -732,7 +732,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
                 .hint = hintfmt(
                     "path '%s' was modified! expected hash '%s', got '%s'",
                     store->printStorePath(path),
-                    info->narHash->to_string(Base32, true),
+                    info->narHash.to_string(Base32, true),
                     current.first.to_string(Base32, true))
             });
             status = 1;
@@ -862,7 +862,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         out << info->narSize // downloadSize
                             << info->narSize;
                         if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << (info->narHash ? info->narHash->to_string(Base32, true) : "")
+                            out << (info->narHash ? info->narHash.to_string(Base32, true) : "")
                                 << renderContentAddress(info->ca)
                                 << info->sigs;
                     } catch (InvalidPath &) {
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 2fe2e2fb2..712043978 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -84,7 +84,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             info.narSize = sink.s->size();
             info.ca = FixedOutputHash {
                 .method = FileIngestionMethod::Recursive,
-                .hash = *info.narHash,
+                .hash = info.narHash,
             };
 
             if (!json)
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 7dcc0b6d4..c6cd88c49 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -133,7 +133,7 @@ struct ProfileManifest
         info.references = std::move(references);
         info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = *info.narHash };
+        info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
 
         auto source = StringSource { *sink.s };
         store->addToStore(info, source);
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index fc7a9765c..26f755fd9 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -91,15 +91,15 @@ struct CmdVerify : StorePathsCommand
 
                     std::unique_ptr<AbstractHashSink> hashSink;
                     if (!info->ca)
-                        hashSink = std::make_unique<HashSink>(info->narHash->type);
+                        hashSink = std::make_unique<HashSink>(info->narHash.type);
                     else
-                        hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
+                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                     store->narFromPath(info->path, *hashSink);
 
                     auto hash = hashSink->finish();
 
-                    if (hash.first != *info->narHash) {
+                    if (hash.first != info->narHash) {
                         corrupted++;
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
                         logError({
@@ -107,7 +107,7 @@ struct CmdVerify : StorePathsCommand
                             .hint = hintfmt(
                                 "path '%s' was modified! expected hash '%s', got '%s'",
                                 store->printStorePath(info->path),
-                                info->narHash->to_string(Base32, true),
+                                info->narHash.to_string(Base32, true),
                                 hash.first.to_string(Base32, true))
                         });
                     }

From 1ad6394b33b5e627c27bc26247f8a5e1d7d81ce1 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Wed, 5 Aug 2020 15:11:49 -0400
Subject: [PATCH 064/882] Add Hash::dummy to signal default value

We did this in the same spirit of the dummy value that's present in
libstore/path.hh
---
 src/libstore/legacy-ssh-store.cc | 2 +-
 src/libstore/path-info.hh        | 4 ++--
 src/libutil/hash.cc              | 2 ++
 src/libutil/hash.hh              | 2 ++
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index ed57a188b..f0f5ec626 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
 
             if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
                 auto s = readString(conn->from);
-                info->narHash = s.empty() ? Hash(htSHA256) : Hash::parseAnyPrefixed(s);
+                info->narHash = s.empty() ? Hash::dummy : Hash::parseAnyPrefixed(s);
                 info->ca = parseContentAddressOpt(readString(conn->from));
                 info->sigs = readStrings<StringSet>(conn->from);
             }
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index ba154dab7..15484717e 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -100,8 +100,8 @@ struct ValidPathInfo
 
     ValidPathInfo(const ValidPathInfo & other) = default;
 
-    ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash(htSHA256)) { };
-    ValidPathInfo(const StorePath & path) : path(path), narHash(Hash(htSHA256)) { };
+    ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash::dummy) { };
+    ValidPathInfo(const StorePath & path) : path(path), narHash(Hash::dummy) { };
 
     virtual ~ValidPathInfo() { }
 };
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index dfb3668f1..4a94f0dfd 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -136,6 +136,8 @@ std::string Hash::to_string(Base base, bool includeType) const
     return s;
 }
 
+Hash Hash::dummy(htSHA256);
+
 Hash Hash::parseSRI(std::string_view original) {
     auto rest = original;
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 00ce7bb6f..0520c6022 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -105,6 +105,8 @@ public:
         assert(type == htSHA1);
         return std::string(to_string(Base16, false), 0, 7);
     }
+
+    static Hash dummy;
 };
 
 /* Helper that defaults empty hashes to the 0 hash. */

From 8241e660bac5583d2924a490e5b32027e9e8e2fc Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Wed, 5 Aug 2020 15:30:38 -0400
Subject: [PATCH 065/882] Remove Hash::operator bool ()

Since the hash is not optional anymore
---
 src/libstore/binary-cache-store.cc |  2 +-
 src/libstore/local-store.cc        |  5 +----
 src/libstore/nar-info.cc           |  4 ++--
 src/libstore/store-api.cc          | 18 ++----------------
 src/libutil/hash.hh                |  3 ---
 src/nix-store/nix-store.cc         |  4 ++--
 6 files changed, 8 insertions(+), 28 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 9682db730..4bcb2c93b 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -143,7 +143,7 @@ struct FileSource : FdSource
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    assert(info.narHash && info.narSize);
+    assert(info.narSize);
 
     if (!repair && isValidPath(info.path)) {
         // FIXME: copyNAR -> null sink
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 3542904bf..fc0630cae 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -920,7 +920,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         StorePathSet paths;
 
         for (auto & i : infos) {
-            assert(i.narHash && i.narHash.type == htSHA256);
+            assert(i.narHash.type == htSHA256);
             if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
             else
@@ -984,9 +984,6 @@ const PublicKeys & LocalStore::getPublicKeys()
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    if (!info.narHash)
-        throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
-
     if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
         throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
 
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index cadaf6267..3a9121679 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -76,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
 
     if (compression == "") compression = "bzip2";
 
-    if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt();
+    if (!havePath || url.empty() || narSize == 0) throw corrupt();
 }
 
 std::string NarInfo::to_string(const Store & store) const
@@ -89,7 +89,7 @@ std::string NarInfo::to_string(const Store & store) const
     assert(fileHash && fileHash->type == htSHA256);
     res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
-    assert(narHash && narHash.type == htSHA256);
+    assert(narHash.type == htSHA256);
     res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index aead3468a..997af1195 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -725,20 +725,6 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
         info = info2;
     }
 
-    if (!info->narHash) {
-        StringSink sink;
-        srcStore->narFromPath({storePath}, sink);
-        auto info2 = make_ref<ValidPathInfo>(*info);
-        info2->narHash = hashString(htSHA256, *sink.s);
-        if (!info->narSize) info2->narSize = sink.s->size();
-        if (info->ultimate) info2->ultimate = false;
-        info = info2;
-
-        StringSource source(*sink.s);
-        dstStore->addToStore(*info, source, repair, checkSigs);
-        return;
-    }
-
     if (info->ultimate) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->ultimate = false;
@@ -910,8 +896,8 @@ string showPaths(const PathSet & paths)
 
 std::string ValidPathInfo::fingerprint(const Store & store) const
 {
-    if (narSize == 0 || !narHash)
-        throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known",
+    if (narSize == 0)
+        throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
             store.printStorePath(path));
     return
         "1;" + store.printStorePath(path) + ";"
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 0520c6022..6d6eb70ca 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -59,9 +59,6 @@ private:
     Hash(std::string_view s, HashType type, bool isSRI);
 
 public:
-    /* Check whether a hash is set. */
-    operator bool () const { return (bool) type; }
-
     /* Check whether two hash are equal. */
     bool operator == (const Hash & h2) const;
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index a901261f4..cc104bf35 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -372,7 +372,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
-                        assert(info->narHash && info->narHash.type == htSHA256);
+                        assert(info->narHash.type == htSHA256);
                         cout << fmt("%s\n", info->narHash.to_string(Base32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
@@ -862,7 +862,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         out << info->narSize // downloadSize
                             << info->narSize;
                         if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << (info->narHash ? info->narHash.to_string(Base32, true) : "")
+                            out << info->narHash.to_string(Base32, true)
                                 << renderContentAddress(info->ca)
                                 << info->sigs;
                     } catch (InvalidPath &) {

From ed96e603e116123c1e44f5108b67b472b2c96538 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 5 Aug 2020 19:44:08 +0000
Subject: [PATCH 066/882] Proxy -> Phantom to match Rust

Sorry, Haskell.
---
 src/libstore/remote-store.cc    |  4 ++--
 src/libstore/worker-protocol.hh | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 377c81ff4..2c0857466 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -58,7 +58,7 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
 }
 
 
-StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
 {
     return store.parseStorePath(readString(from));
 }
@@ -461,7 +461,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryDerivationOutp
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputMap << printStorePath(path);
     conn.processStderr();
-    return read(*this, conn->from, Proxy<std::map<std::string, std::optional<StorePath>>> {});
+    return read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
 
 }
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 117d3e1a4..384b81f08 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -72,16 +72,16 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
 
 /* To guide overloading */
 template<typename T>
-struct Proxy {};
+struct Phantom {};
 
 template<typename T>
-std::map<std::string, T> read(const Store & store, Source & from, Proxy<std::map<std::string, T>> _)
+std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _)
 {
     std::map<string, T> resMap;
     auto size = (size_t)readInt(from);
     while (size--) {
         auto thisKey = readString(from);
-        resMap.insert_or_assign(std::move(thisKey), read(store, from, Proxy<T> {}));
+        resMap.insert_or_assign(std::move(thisKey), read(store, from, Phantom<T> {}));
     }
     return resMap;
 }
@@ -97,14 +97,14 @@ void write(const Store & store, Sink & out, const std::map<string, T> & resMap)
 }
 
 template<typename T>
-std::optional<T> read(const Store & store, Source & from, Proxy<std::optional<T>> _)
+std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _)
 {
     auto tag = readNum<uint8_t>(from);
     switch (tag) {
     case 0:
         return std::nullopt;
     case 1:
-        return read(store, from, Proxy<T> {});
+        return read(store, from, Phantom<T> {});
     default:
         throw Error("got an invalid tag bit for std::optional: %#04x", tag);
     }
@@ -118,7 +118,7 @@ void write(const Store & store, Sink & out, const std::optional<T> & optVal)
         write(store, out, *optVal);
 }
 
-StorePath read(const Store & store, Source & from, Proxy<StorePath> _);
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _);
 
 void write(const Store & store, Sink & out, const StorePath & storePath);
 

From d5d9907a9ebd89e84d5031cd767dd4d97145803a Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Wed, 5 Aug 2020 15:57:42 -0400
Subject: [PATCH 067/882] Fix perl integration

---
 perl/lib/Nix/Store.xs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index de60fb939..41846e6a7 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);

From 6c66331d5c34d97092a9b5e2832fed73e0da3c87 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 5 Aug 2020 20:37:48 +0000
Subject: [PATCH 068/882] WIP: Put the worker protocol `read` and `write` in a
 namespace to disambig

---
 src/libstore/remote-store.cc    |  6 +++++-
 src/libstore/worker-protocol.hh | 15 +++++++++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 2c0857466..96e0aabba 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -58,6 +58,8 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
 }
 
 
+namespace worker_proto {
+
 StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
 {
     return store.parseStorePath(readString(from));
@@ -68,6 +70,8 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
     out << store.printStorePath(storePath);
 }
 
+}
+
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -461,7 +465,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryDerivationOutp
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputMap << printStorePath(path);
     conn.processStderr();
-    return read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+    return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
 
 }
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 384b81f08..fdd700668 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -74,6 +74,10 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
 template<typename T>
 struct Phantom {};
 
+
+namespace worker_proto {
+/* FIXME maybe move more stuff inside here */
+
 template<typename T>
 std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _)
 {
@@ -81,7 +85,7 @@ std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::m
     auto size = (size_t)readInt(from);
     while (size--) {
         auto thisKey = readString(from);
-        resMap.insert_or_assign(std::move(thisKey), read(store, from, Phantom<T> {}));
+        resMap.insert_or_assign(std::move(thisKey), nix::worker_proto::read(store, from, Phantom<T> {}));
     }
     return resMap;
 }
@@ -92,7 +96,7 @@ void write(const Store & store, Sink & out, const std::map<string, T> & resMap)
     out << resMap.size();
     for (auto & i : resMap) {
         out << i.first;
-        write(store, out, i.second);
+        nix::worker_proto::write(store, out, i.second);
     }
 }
 
@@ -104,7 +108,7 @@ std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<
     case 0:
         return std::nullopt;
     case 1:
-        return read(store, from, Phantom<T> {});
+        return nix::worker_proto::read(store, from, Phantom<T> {});
     default:
         throw Error("got an invalid tag bit for std::optional: %#04x", tag);
     }
@@ -115,13 +119,16 @@ void write(const Store & store, Sink & out, const std::optional<T> & optVal)
 {
     out << (optVal ? 1 : 0);
     if (optVal)
-        write(store, out, *optVal);
+        nix::worker_proto::write(store, out, *optVal);
 }
 
 StorePath read(const Store & store, Source & from, Phantom<StorePath> _);
 
 void write(const Store & store, Sink & out, const StorePath & storePath);
 
+}
+
+
 StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
 
 void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);

From 0739d428e07b59b7d44bdbf7354dafaae09dba7c Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Wed, 5 Aug 2020 17:49:45 -0400
Subject: [PATCH 069/882] Solve template deduction problem

We had to predeclare our template functions
---
 src/libstore/daemon.cc          |  2 +-
 src/libstore/worker-protocol.hh | 16 +++++++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index e18f793c8..a420dcab0 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -327,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryDerivationOutputMap(path);
         logger->stopWork();
-        write(*store, to, outputs);
+        nix::worker_proto::write(*store, to, outputs);
         break;
     }
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index fdd700668..c50995d7c 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -78,6 +78,19 @@ struct Phantom {};
 namespace worker_proto {
 /* FIXME maybe move more stuff inside here */
 
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _);
+void write(const Store & store, Sink & out, const StorePath & storePath);
+
+template<typename T>
+std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _);
+template<typename T>
+void write(const Store & store, Sink & out, const std::map<string, T> & resMap);
+template<typename T>
+std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _);
+template<typename T>
+void write(const Store & store, Sink & out, const std::optional<T> & optVal);
+
+
 template<typename T>
 std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _)
 {
@@ -122,9 +135,6 @@ void write(const Store & store, Sink & out, const std::optional<T> & optVal)
         nix::worker_proto::write(store, out, *optVal);
 }
 
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _);
-
-void write(const Store & store, Sink & out, const StorePath & storePath);
 
 }
 

From 8b175f58d1a25dd904fb0ffdf5b2b9501983dba0 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Wed, 5 Aug 2020 17:57:07 -0400
Subject: [PATCH 070/882] Simplify the namespace

---
 src/libstore/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index a420dcab0..9503915eb 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -327,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryDerivationOutputMap(path);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, outputs);
+        worker_proto::write(*store, to, outputs);
         break;
     }
 

From 59067f0f587f75908c4f990bcbab29340c7f7652 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 6 Aug 2020 11:40:41 +0200
Subject: [PATCH 071/882] repl.cc: Check for HAVE_BOEHMGC

Fixes #3906.
---
 src/nix/repl.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index fb9050d0d..c3c9e54a8 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -33,12 +33,17 @@ extern "C" {
 #include "command.hh"
 #include "finally.hh"
 
+#if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
 #include <gc/gc_cpp.h>
+#endif
 
 namespace nix {
 
-struct NixRepl : gc
+struct NixRepl
+    #if HAVE_BOEHMGC
+    : gc
+    #endif
 {
     string curDir;
     std::unique_ptr<EvalState> state;

From e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 6 Aug 2020 18:31:48 +0000
Subject: [PATCH 072/882] Minimize the usage of `Hash::dummy`

---
 src/libfetchers/tarball.cc          |  6 ++++--
 src/libstore/binary-cache-store.cc  | 10 ++++++++--
 src/libstore/build.cc               |  6 ++++--
 src/libstore/daemon.cc              |  5 +++--
 src/libstore/export-import.cc       | 11 ++++++-----
 src/libstore/legacy-ssh-store.cc    | 19 +++++++++++++------
 src/libstore/local-store.cc         | 21 +++++++++++----------
 src/libstore/nar-info-disk-cache.cc |  5 +++--
 src/libstore/nar-info.cc            | 10 +++++++---
 src/libstore/nar-info.hh            |  6 ++++--
 src/libstore/path-info.hh           |  5 +++--
 src/libstore/remote-store.cc        |  4 ++--
 src/libstore/store-api.cc           | 19 ++++++++++++-------
 src/libstore/store-api.hh           |  3 +--
 src/nix-store/nix-store.cc          | 11 ++++++++---
 src/nix/add-to-store.cc             |  6 ++++--
 src/nix/make-content-addressable.cc |  6 ++++--
 src/nix/profile.cc                  |  6 ++++--
 18 files changed, 101 insertions(+), 58 deletions(-)

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 55158cece..a2d16365e 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -67,8 +67,10 @@ DownloadFileResult downloadFile(
         StringSink sink;
         dumpString(*res.data, sink);
         auto hash = hashString(htSHA256, *res.data);
-        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
-        info.narHash = hashString(htSHA256, *sink.s);
+        ValidPathInfo info {
+            store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
+            hashString(htSHA256, *sink.s),
+        };
         info.narSize = sink.s->size();
         info.ca = FixedOutputHash {
             .method = FileIngestionMethod::Flat,
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4bcb2c93b..9608467e2 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -385,7 +385,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info(makeFixedOutputPath(method, *h, name));
+    ValidPathInfo info {
+        makeFixedOutputPath(method, *h, name),
+        Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+    };
 
     auto source = StringSource { *sink.s };
     addToStore(info, source, repair, CheckSigs);
@@ -396,7 +399,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
 StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
-    ValidPathInfo info(computeStorePathForText(name, s, references));
+    ValidPathInfo info {
+        computeStorePathForText(name, s, references),
+        Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+    };
     info.references = references;
 
     if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 3370af63d..9effd7ec0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3864,8 +3864,10 @@ void DerivationGoal::registerOutputs()
             worker.markContentsGood(worker.store.parseStorePath(path));
         }
 
-        ValidPathInfo info(worker.store.parseStorePath(path));
-        info.narHash = hash.first;
+        ValidPathInfo info {
+            worker.store.parseStorePath(path),
+            hash.first,
+        };
         info.narSize = hash.second;
         info.references = std::move(references);
         info.deriver = drvPath;
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index fa3bf6e2d..7f7f25ebf 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -694,11 +694,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopAddToStoreNar: {
         bool repair, dontCheckSigs;
-        ValidPathInfo info(store->parseStorePath(readString(from)));
+        auto path = store->parseStorePath(readString(from));
         auto deriver = readString(from);
+        auto narHash = Hash::parseAny(readString(from), htSHA256);
+        ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.narHash = Hash::parseAny(readString(from), htSHA256);
         info.references = readStorePaths<StorePathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 3f4b7c36f..ccd466d09 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -69,17 +69,18 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         if (magic != exportMagic)
             throw Error("Nix archive cannot be imported; wrong format");
 
-        ValidPathInfo info(parseStorePath(readString(source)));
+        auto path = parseStorePath(readString(source));
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        info.references = readStorePaths<StorePathSet>(*this, source);
-
+        auto references = readStorePaths<StorePathSet>(*this, source);
         auto deriver = readString(source);
+        auto narHash = hashString(htSHA256, *saved.s);
+
+        ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
-
-        info.narHash = hashString(htSHA256, *saved.s);
+        info.references = references;
         info.narSize = saved.s->size();
 
         // Ignore optional legacy signature.
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index f0f5ec626..9f95a9726 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -93,6 +93,9 @@ struct LegacySSHStore : public Store
         try {
             auto conn(connections->get());
 
+            /* No longer support missing NAR hash */
+            assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
+
             debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
 
             conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)};
@@ -100,8 +103,10 @@ struct LegacySSHStore : public Store
 
             auto p = readString(conn->from);
             if (p.empty()) return callback(nullptr);
-            auto info = std::make_shared<ValidPathInfo>(parseStorePath(p));
-            assert(path == info->path);
+            auto path2 = parseStorePath(p);
+            assert(path == path2);
+            /* Hash will be set below. FIXME construct ValidPathInfo at end. */
+            auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy);
 
             PathSet references;
             auto deriver = readString(conn->from);
@@ -111,12 +116,14 @@ struct LegacySSHStore : public Store
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
-            if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
+            {
                 auto s = readString(conn->from);
-                info->narHash = s.empty() ? Hash::dummy : Hash::parseAnyPrefixed(s);
-                info->ca = parseContentAddressOpt(readString(conn->from));
-                info->sigs = readStrings<StringSet>(conn->from);
+                if (s == "")
+                    throw Error("NAR hash is now mandatory");
+                info->narHash = Hash::parseAnyPrefixed(s);
             }
+            info->ca = parseContentAddressOpt(readString(conn->from));
+            info->sigs = readStrings<StringSet>(conn->from);
 
             auto s = readString(conn->from);
             assert(s == "");
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index fc0630cae..c96db68ed 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -641,25 +641,28 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto info = std::make_shared<ValidPathInfo>(path);
-
         callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
 
             /* Get the path info. */
-            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path)));
+            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path)));
 
             if (!useQueryPathInfo.next())
                 return std::shared_ptr<ValidPathInfo>();
 
-            info->id = useQueryPathInfo.getInt(0);
+            auto id = useQueryPathInfo.getInt(0);
 
+            auto narHash = Hash::dummy;
             try {
-                info->narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
+                narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
             } catch (BadHash & e) {
-                throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
+                throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
             }
 
+            auto info = std::make_shared<ValidPathInfo>(path, narHash);
+
+            info->id = id;
+
             info->registrationTime = useQueryPathInfo.getInt(2);
 
             auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
@@ -1152,8 +1155,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
 
             optimisePath(realPath);
 
-            ValidPathInfo info(dstPath);
-            info.narHash = narHash.first;
+            ValidPathInfo info { dstPath, narHash.first };
             info.narSize = narHash.second;
             info.ca = FixedOutputHash { .method = method, .hash = hash };
             registerValidPath(info);
@@ -1196,8 +1198,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
             optimisePath(realPath);
 
-            ValidPathInfo info(dstPath);
-            info.narHash = narHash;
+            ValidPathInfo info { dstPath, narHash };
             info.narSize = sink.s->size();
             info.references = references;
             info.ca = TextHash { .hash = hash };
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index dd40006d7..8541cc51f 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -189,13 +189,14 @@ public:
                 return {oInvalid, 0};
 
             auto namePart = queryNAR.getStr(1);
-            auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart));
+            auto narInfo = make_ref<NarInfo>(
+                StorePath(hashPart + "-" + namePart),
+                Hash::parseAnyPrefixed(queryNAR.getStr(6)));
             narInfo->url = queryNAR.getStr(2);
             narInfo->compression = queryNAR.getStr(3);
             if (!queryNAR.isNull(4))
                 narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4));
             narInfo->fileSize = queryNAR.getInt(5);
-            narInfo->narHash = Hash::parseAnyPrefixed(queryNAR.getStr(6));
             narInfo->narSize = queryNAR.getInt(7);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
                 narInfo->references.insert(StorePath(r));
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 3a9121679..3454f34bb 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -1,10 +1,11 @@
 #include "globals.hh"
 #include "nar-info.hh"
+#include "store-api.hh"
 
 namespace nix {
 
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
-    : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
+    : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack
 {
     auto corrupt = [&]() {
         return Error("NAR info file '%1%' is corrupt", whence);
@@ -19,6 +20,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
     };
 
     bool havePath = false;
+    bool haveNarHash = false;
 
     size_t pos = 0;
     while (pos < s.size()) {
@@ -46,8 +48,10 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "FileSize") {
             if (!string2Int(value, fileSize)) throw corrupt();
         }
-        else if (name == "NarHash")
+        else if (name == "NarHash") {
             narHash = parseHashField(value);
+            haveNarHash = true;
+        }
         else if (name == "NarSize") {
             if (!string2Int(value, narSize)) throw corrupt();
         }
@@ -76,7 +80,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
 
     if (compression == "") compression = "bzip2";
 
-    if (!havePath || url.empty() || narSize == 0) throw corrupt();
+    if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt();
 }
 
 std::string NarInfo::to_string(const Store & store) const
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index eff19f0ef..39ced76e5 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -2,10 +2,12 @@
 
 #include "types.hh"
 #include "hash.hh"
-#include "store-api.hh"
+#include "path-info.hh"
 
 namespace nix {
 
+class Store;
+
 struct NarInfo : ValidPathInfo
 {
     std::string url;
@@ -15,7 +17,7 @@ struct NarInfo : ValidPathInfo
     std::string system;
 
     NarInfo() = delete;
-    NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { }
+    NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
     NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
     NarInfo(const Store & store, const std::string & s, const std::string & whence);
 
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 15484717e..8ff5c466e 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "crypto.hh"
 #include "path.hh"
 #include "hash.hh"
 #include "content-address.hh"
@@ -100,8 +101,8 @@ struct ValidPathInfo
 
     ValidPathInfo(const ValidPathInfo & other) = default;
 
-    ValidPathInfo(StorePath && path) : path(std::move(path)), narHash(Hash::dummy) { };
-    ValidPathInfo(const StorePath & path) : path(path), narHash(Hash::dummy) { };
+    ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
+    ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
 
     virtual ~ValidPathInfo() { }
 };
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b5e4d9fa6..8dcc1d710 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -419,10 +419,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
                 bool valid; conn->from >> valid;
                 if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
             }
-            info = std::make_shared<ValidPathInfo>(StorePath(path));
             auto deriver = readString(conn->from);
+            auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
+            info = std::make_shared<ValidPathInfo>(path, narHash);
             if (deriver != "") info->deriver = parseStorePath(deriver);
-            info->narHash = Hash::parseAny(readString(conn->from), htSHA256);
             info->references = readStorePaths<StorePathSet>(*this, conn->from);
             conn->from >> info->registrationTime >> info->narSize;
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 997af1195..5a061b2c9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -320,8 +320,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     if (expectedCAHash && expectedCAHash != hash)
         throw Error("hash mismatch for '%s'", srcPath);
 
-    ValidPathInfo info(makeFixedOutputPath(method, hash, name));
-    info.narHash = narHash;
+    ValidPathInfo info {
+        makeFixedOutputPath(method, hash, name),
+        narHash,
+    };
     info.narSize = narSize;
     info.ca = FixedOutputHash { .method = method, .hash = hash };
 
@@ -849,19 +851,22 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
 }
 
 
-std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven)
+std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
 {
     std::string path;
     getline(str, path);
     if (str.eof()) { return {}; }
-    ValidPathInfo info(store.parseStorePath(path));
-    if (hashGiven) {
+    if (!hashGiven) {
         string s;
         getline(str, s);
-        info.narHash = Hash::parseAny(s, htSHA256);
+        auto narHash = Hash::parseAny(s, htSHA256);
         getline(str, s);
-        if (!string2Int(s, info.narSize)) throw Error("number expected");
+        uint64_t narSize;
+        if (!string2Int(s, narSize)) throw Error("number expected");
+        hashGiven = { narHash, narSize };
     }
+    ValidPathInfo info(store.parseStorePath(path), hashGiven->first);
+    info.narSize = hashGiven->second;
     std::string deriver;
     getline(str, deriver);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e94d975c5..f6f6acaf5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -4,7 +4,6 @@
 #include "hash.hh"
 #include "content-address.hh"
 #include "serialise.hh"
-#include "crypto.hh"
 #include "lru-cache.hh"
 #include "sync.hh"
 #include "globals.hh"
@@ -761,7 +760,7 @@ string showPaths(const PathSet & paths);
 std::optional<ValidPathInfo> decodeValidPathInfo(
     const Store & store,
     std::istream & str,
-    bool hashGiven = false);
+    std::optional<HashResult> hashGiven = std::nullopt);
 
 /* Split URI into protocol+hierarchy part and its parameter set. */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index cc104bf35..e08d908e6 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -495,7 +495,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
     ValidPathInfos infos;
 
     while (1) {
-        auto info = decodeValidPathInfo(*store, cin, hashGiven);
+        // We use a dummy value because we'll set it below. FIXME be correct by
+        // construction and avoid dummy value.
+        auto hashResultOpt = !hashGiven ? std::optional<HashResult> { {Hash::dummy, -1} } : std::nullopt;
+        auto info = decodeValidPathInfo(*store, cin, hashResultOpt);
         if (!info) break;
         if (!store->isValidPath(info->path) || reregister) {
             /* !!! races */
@@ -944,11 +947,13 @@ static void opServe(Strings opFlags, Strings opArgs)
                 if (!writeAllowed) throw Error("importing paths is not allowed");
 
                 auto path = readString(in);
-                ValidPathInfo info(store->parseStorePath(path));
                 auto deriver = readString(in);
+                ValidPathInfo info {
+                    store->parseStorePath(path),
+                    Hash::parseAny(readString(in), htSHA256),
+                };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.narHash = Hash::parseAny(readString(in), htSHA256);
                 info.references = readStorePaths<StorePathSet>(*store, in);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index e183cb8b5..713155840 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -60,8 +60,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             hash = hsink.finish().first;
         }
 
-        ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
-        info.narHash = narHash;
+        ValidPathInfo info {
+            store->makeFixedOutputPath(ingestionMethod, hash, *namePart),
+            narHash,
+        };
         info.narSize = sink.s->size();
         info.ca = std::optional { FixedOutputHash {
             .method = ingestionMethod,
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 712043978..38b60fc38 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -77,10 +77,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
             auto narHash = hashModuloSink.finish().first;
 
-            ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference));
+            ValidPathInfo info {
+                store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
+                narHash,
+            };
             info.references = std::move(references);
             if (hasSelfReference) info.references.insert(info.path);
-            info.narHash = narHash;
             info.narSize = sink.s->size();
             info.ca = FixedOutputHash {
                 .method = FileIngestionMethod::Recursive,
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index c6cd88c49..cffc9ee44 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -129,9 +129,11 @@ struct ProfileManifest
 
         auto narHash = hashString(htSHA256, *sink.s);
 
-        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references));
+        ValidPathInfo info {
+            store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references),
+            narHash,
+        };
         info.references = std::move(references);
-        info.narHash = narHash;
         info.narSize = sink.s->size();
         info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
 

From 3d8240c32eedd0809450be52e4ac7625ffdad9aa Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Thu, 6 Aug 2020 16:04:18 -0400
Subject: [PATCH 073/882] Remove leftover commented code

---
 src/libstore/worker-protocol.hh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b4c260e26..3bb27ab22 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -136,7 +136,6 @@ void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
 {
     out << resMap.size();
     for (auto & i : resMap) {
-        // out << i.first;
         write(store, out, i.first);
         write(store, out, i.second);
     }
@@ -150,7 +149,6 @@ std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<
     case 0:
         return std::nullopt;
     case 1:
-        // return nix::worker_proto::read(store, from, Phantom<T> {});
         return read(store, from, Phantom<T> {});
     default:
         throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);

From 9ab07e99f527d1fa3adfa02839da477a1528d64b Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Thu, 6 Aug 2020 18:04:13 -0400
Subject: [PATCH 074/882] Use template structs instead of phantoms

---
 src/libstore/build.cc            |   4 +-
 src/libstore/daemon.cc           |  36 +++----
 src/libstore/derivations.cc      |   4 +-
 src/libstore/export-import.cc    |   4 +-
 src/libstore/legacy-ssh-store.cc |  14 +--
 src/libstore/remote-store.cc     |  61 ++++++------
 src/libstore/worker-protocol.hh  | 165 +++++++++++++++----------------
 src/nix-store/nix-store.cc       |  16 +--
 8 files changed, 148 insertions(+), 156 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8340828e7..e1c360338 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1864,11 +1864,11 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    nix::worker_proto::write(worker.store, hook->sink, inputPaths);
+    WorkerProto<StorePathSet>::write(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
-    nix::worker_proto::write(worker.store, hook->sink, missingPaths);
+    WorkerProto<StorePathSet>::write(worker.store, hook->sink, missingPaths);
 
     hook->sink = FdSink();
     hook->toHook.writeSide = -1;
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 6d734abdc..148bd5cc9 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -256,11 +256,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto res = store->queryValidPaths(paths);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, res);
+        WorkerProto<StorePathSet>::write(*store, to, res);
         break;
     }
 
@@ -276,11 +276,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, res);
+        WorkerProto<StorePathSet>::write(*store, to, res);
         break;
     }
 
@@ -309,7 +309,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, paths);
+        WorkerProto<StorePathSet>::write(*store, to, paths);
         break;
     }
 
@@ -327,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryDerivationOutputMap(path);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, outputs);
+        WorkerProto<std::map<std::string, std::optional<StorePath>>>::write(*store, to, outputs);
         break;
     }
 
@@ -397,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopAddTextToStore: {
         string suffix = readString(from);
         string s = readString(from);
-        auto refs = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto refs = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -518,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -587,7 +587,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            nix::worker_proto::write(*store, to, i->second.references);
+            WorkerProto<StorePathSet>::write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -598,11 +598,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto paths = WorkerProto<StorePathSet>::read(*store, from);
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = nix::worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
+            pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -610,7 +610,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            nix::worker_proto::write(*store, to, i.second.references);
+            WorkerProto<StorePathSet>::write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        nix::worker_proto::write(*store, to, paths);
+        WorkerProto<StorePathSet>::write(*store, to, paths);
         break;
     }
 
@@ -639,7 +639,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 to << 1;
             to << (info->deriver ? store->printStorePath(*info->deriver) : "")
                << info->narHash->to_string(Base16, false);
-            nix::worker_proto::write(*store, to, info->references);
+            WorkerProto<StorePathSet>::write(*store, to, info->references);
             to << info->registrationTime << info->narSize;
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
@@ -699,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
         info.narHash = Hash::parseAny(readString(from), htSHA256);
-        info.references = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        info.references = WorkerProto<StorePathSet>::read(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = parseContentAddressOpt(readString(from));
@@ -799,9 +799,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, willBuild);
-        nix::worker_proto::write(*store, to, willSubstitute);
-        nix::worker_proto::write(*store, to, unknown);
+        WorkerProto<StorePathSet>::write(*store, to, willBuild);
+        WorkerProto<StorePathSet>::write(*store, to, willSubstitute);
+        WorkerProto<StorePathSet>::write(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index b7aef9d65..bf2758ae5 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -605,7 +605,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = nix::worker_proto::read(store, in, Phantom<StorePathSet> {});
+    drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -640,7 +640,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.output);
     }
-    nix::worker_proto::write(store, out, drv.inputSrcs);
+    WorkerProto<StorePathSet>::write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 7551294f7..e2e52b0af 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    nix::worker_proto::write(*this, teeSink, info->references);
+    WorkerProto<StorePathSet>::write(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        info.references = nix::worker_proto::read(*this, source, Phantom<StorePathSet> {});
+        info.references = WorkerProto<StorePathSet>::read(*this, source);
 
         auto deriver = readString(source);
         if (deriver != "")
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 0951610d3..ee07b7156 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -107,7 +107,7 @@ struct LegacySSHStore : public Store
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -139,7 +139,7 @@ struct LegacySSHStore : public Store
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash->to_string(Base16, false);
-            nix::worker_proto::write(*this, conn->to, info.references);
+            WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -168,7 +168,7 @@ struct LegacySSHStore : public Store
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            nix::worker_proto::write(*this, conn->to, info.references);
+            WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -251,10 +251,10 @@ struct LegacySSHStore : public Store
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        nix::worker_proto::write(*this, conn->to, paths);
+        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+        for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
             out.insert(i);
     }
 
@@ -267,10 +267,10 @@ struct LegacySSHStore : public Store
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        nix::worker_proto::write(*this, conn->to, paths);
+        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
         conn->to.flush();
 
-        return nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 
     void connect() override
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index ac776b95a..48450eb67 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -22,40 +22,36 @@
 
 namespace nix {
 
-namespace worker_proto {
-
-std::string read(const Store & store, Source & from, Phantom<std::string> _)
+std::string WorkerProto<std::string>::read(const Store & store, Source & from)
 {
     return readString(from);
 }
 
-void write(const Store & store, Sink & out, const std::string & str)
+void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
 {
     out << str;
 }
 
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
+StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
 {
     return store.parseStorePath(readString(from));
 }
 
-void write(const Store & store, Sink & out, const StorePath & storePath)
+void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
 {
     out << store.printStorePath(storePath);
 }
 
-ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
+ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
 {
     return parseContentAddress(readString(from));
 }
 
-void write(const Store & store, Sink & out, const ContentAddress & ca)
+void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
 {
     out << renderContentAddress(ca);
 }
 
-}
-
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -290,9 +286,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        nix::worker_proto::write(*this, conn->to, paths);
+        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -302,7 +298,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -319,9 +315,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        nix::worker_proto::write(*this, conn->to, paths);
+        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -343,7 +339,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -356,9 +352,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            worker_proto::write(*this, conn->to, paths);
+            WorkerProto<StorePathSet>::write(*this, conn->to, paths);
         } else
-            worker_proto::write(*this, conn->to, pathsMap);
+            WorkerProto<StorePathCAMap>::write(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -366,7 +362,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -399,7 +395,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
             auto deriver = readString(conn->from);
             if (deriver != "") info->deriver = parseStorePath(deriver);
             info->narHash = Hash::parseAny(readString(conn->from), htSHA256);
-            info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
             conn->from >> info->registrationTime >> info->narSize;
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
                 conn->from >> info->ultimate;
@@ -418,7 +414,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+    for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
         referrers.insert(i);
 }
 
@@ -428,7 +424,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -440,7 +436,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -449,8 +445,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryDerivationOutp
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputMap << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
-
+    return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
 }
 
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
@@ -479,7 +474,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, sink, info.references);
+            WorkerProto<StorePathSet>::write(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -489,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
         assert(importedPaths.size() <= 1);
     }
 
@@ -498,7 +493,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash->to_string(Base16, false);
-        worker_proto::write(*this, conn->to, info.references);
+        WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -631,7 +626,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
 
     auto conn(getConnection());
     conn->to << wopAddTextToStore << name << s;
-    worker_proto::write(*this, conn->to, references);
+    WorkerProto<StorePathSet>::write(*this, conn->to, references);
 
     conn.processStderr();
     return parseStorePath(readString(conn->from));
@@ -733,7 +728,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << wopCollectGarbage << options.action;
-    worker_proto::write(*this, conn->to, options.pathsToDelete);
+    WorkerProto<StorePathSet>::write(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -795,9 +790,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
             ss.push_back(p.to_string(*this));
         conn->to << ss;
         conn.processStderr();
-        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
+        willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
+        unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
         conn->from >> downloadSize >> narSize;
         return;
     }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 3bb27ab22..60543d626 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -66,105 +66,102 @@ typedef enum {
 class Store;
 struct Source;
 
-/* To guide overloading */
 template<typename T>
-struct Phantom {};
+struct WorkerProto {
+    static T read(const Store & store, Source & from);
+    static void write(const Store & store, Sink & out, const T & t);
+};
 
+template<>
+struct WorkerProto<std::string> {
+    static std::string read(const Store & store, Source & from);
+    static void write(const Store & store, Sink & out, const std::string & t);
+};
 
-namespace worker_proto {
-/* FIXME maybe move more stuff inside here */
+template<>
+struct WorkerProto<StorePath> {
+    static StorePath read(const Store & store, Source & from);
+    static void write(const Store & store, Sink & out, const StorePath & t);
+};
 
-std::string read(const Store & store, Source & from, Phantom<std::string> _);
-void write(const Store & store, Sink & out, const std::string & str);
-
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _);
-void write(const Store & store, Sink & out, const StorePath & storePath);
-
-ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _);
-void write(const Store & store, Sink & out, const ContentAddress & ca);
+template<>
+struct WorkerProto<ContentAddress> {
+    static ContentAddress read(const Store & store, Source & from);
+    static void write(const Store & store, Sink & out, const ContentAddress & t);
+};
 
 template<typename T>
-std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _);
-template<typename T>
-void write(const Store & store, Sink & out, const std::set<T> & resSet);
+struct WorkerProto<std::set<T>> {
+
+    static std::set<T> read(const Store & store, Source & from)
+    {
+        std::set<T> resSet;
+        auto size = readNum<size_t>(from);
+        while (size--) {
+            resSet.insert(WorkerProto<T>::read(store, from));
+        }
+        return resSet;
+    }
+
+    static void write(const Store & store, Sink & out, const std::set<T> & resSet)
+    {
+        out << resSet.size();
+        for (auto & key : resSet) {
+            WorkerProto<T>::write(store, out, key);
+        }
+    }
+
+};
 
 template<typename K, typename V>
-std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _);
-template<typename K, typename V>
-void write(const Store & store, Sink & out, const std::map<K, V> & resMap);
+struct WorkerProto<std::map<K, V>> {
 
-template<typename T>
-std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _);
-template<typename T>
-void write(const Store & store, Sink & out, const std::optional<T> & optVal);
-
-template<typename T>
-std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
-{
-    std::set<T> resSet;
-    auto size = readNum<size_t>(from);
-    while (size--) {
-        resSet.insert(read(store, from, Phantom<T> {}));
+    static std::map<K, V> read(const Store & store, Source & from)
+    {
+        std::map<K, V> resMap;
+        auto size = readNum<size_t>(from);
+        while (size--) {
+            resMap.insert_or_assign(
+                WorkerProto<K>::read(store, from),
+                WorkerProto<V>::read(store, from));
+        }
+        return resMap;
     }
-    return resSet;
-}
+
+    static void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+    {
+        out << resMap.size();
+        for (auto & i : resMap) {
+            WorkerProto<K>::write(store, out, i.first);
+            WorkerProto<V>::write(store, out, i.second);
+        }
+    }
+
+};
 
 template<typename T>
-void write(const Store & store, Sink & out, const std::set<T> & resSet)
-{
-    out << resSet.size();
-    for (auto & key : resSet) {
-        write(store, out, key);
+struct WorkerProto<std::optional<T>> {
+
+    static std::optional<T> read(const Store & store, Source & from)
+    {
+        auto tag = readNum<uint8_t>(from);
+        switch (tag) {
+        case 0:
+            return std::nullopt;
+        case 1:
+            return WorkerProto<T>::read(store, from);
+        default:
+            throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
+        }
     }
-}
 
-template<typename K, typename V>
-std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
-{
-    std::map<K, V> resMap;
-    auto size = readNum<size_t>(from);
-    while (size--) {
-        resMap.insert_or_assign(
-            read(store, from, Phantom<K> {}),
-            read(store, from, Phantom<V> {}));
+    static void write(const Store & store, Sink & out, const std::optional<T> & optVal)
+    {
+        out << (uint64_t) (optVal ? 1 : 0);
+        if (optVal)
+            WorkerProto<T>::write(store, out, *optVal);
     }
-    return resMap;
-}
-
-template<typename K, typename V>
-void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
-{
-    out << resMap.size();
-    for (auto & i : resMap) {
-        write(store, out, i.first);
-        write(store, out, i.second);
-    }
-}
-
-template<typename T>
-std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _)
-{
-    auto tag = readNum<uint8_t>(from);
-    switch (tag) {
-    case 0:
-        return std::nullopt;
-    case 1:
-        return read(store, from, Phantom<T> {});
-    default:
-        throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
-    }
-}
-
-template<typename T>
-void write(const Store & store, Sink & out, const std::optional<T> & optVal)
-{
-    out << (uint64_t) (optVal ? 1 : 0);
-    if (optVal)
-        nix::worker_proto::write(store, out, *optVal);
-}
-
-
-}
 
+};
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index a0007f1c4..e59009d11 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -815,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                auto paths = WorkerProto<StorePathSet>::read(*store, in);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -845,19 +845,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                         }
                 }
 
-                nix::worker_proto::write(*store, out, store->queryValidPaths(paths));
+                WorkerProto<StorePathSet>::write(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                auto paths = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                auto paths = WorkerProto<StorePathSet>::read(*store, in);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        nix::worker_proto::write(*store, out, info->references);
+                        WorkerProto<StorePathSet>::write(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -885,7 +885,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(nix::worker_proto::read(*store, in, Phantom<StorePathSet> {}), out);
+                store->exportPaths(WorkerProto<StorePathSet>::read(*store, in), out);
                 break;
             }
 
@@ -934,9 +934,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(nix::worker_proto::read(*store, in, Phantom<StorePathSet> {}),
+                store->computeFSClosure(WorkerProto<StorePathSet>::read(*store, in),
                     closure, false, includeOutputs);
-                nix::worker_proto::write(*store, out, closure);
+                WorkerProto<StorePathSet>::write(*store, out, closure);
                 break;
             }
 
@@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
                 info.narHash = Hash::parseAny(readString(in), htSHA256);
-                info.references = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                info.references = WorkerProto<StorePathSet>::read(*store, in);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = parseContentAddressOpt(readString(in));

From 641c95070162595c71a1f775a2364cd8533e1c4b Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 6 Aug 2020 18:13:14 -0500
Subject: [PATCH 075/882] Add hashed-mirrors back
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Some users have their own hashed-mirrors setup, that is used to mirror
things in addition to what’s available on tarballs.nixos.org. Although
this should be feasable to do with a Binary Cache, it’s not always
easy, since you have to remember what "name" each of the tarballs has.
Continuing to support hashed-mirrors is cheap, so it’s best to leave
support in Nix. Note that NIX_HASHED_MIRRORS is also supported in
Nixpkgs through fetchurl.nix.

Note that this excludes tarballs.nixos.org from the default, as in
\#3689. All of these are available on cache.nixos.org.
---
 doc/manual/command-ref/conf-file.xml | 27 +++++++++++++++++++++++++++
 src/libstore/builtins/fetchurl.cc    | 14 ++++++++++++++
 src/libstore/globals.hh              |  3 +++
 3 files changed, 44 insertions(+)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 9c55526a3..d0f1b09ca 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,6 +370,33 @@ false</literal>.</para>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
+
+    <listitem><para>A list of web servers used by
+    <function>builtins.fetchurl</function> to obtain files by hash.
+    Given a hash type <replaceable>ht</replaceable> and a base-16 hash
+    <replaceable>h</replaceable>, Nix will try to download the file
+    from
+    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
+    This allows files to be downloaded even if they have disappeared
+    from their original URI. For example, given the hashed mirror
+    <literal>http://tarballs.example.com/</literal>, when building the
+    derivation
+
+<programlisting>
+builtins.fetchurl {
+  url = "https://example.org/foo-1.2.3.tar.xz";
+  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+}
+</programlisting>
+
+    Nix will attempt to download this file from
+    <literal>http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
+    first. If it is not available there, if will try the original URI.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
 
     <listitem><para>The maximum number of parallel TCP connections
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 6585a480d..2048f8f87 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,6 +58,20 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         }
     };
 
+    /* Try the hashed mirrors first. */
+    if (getAttr("outputHashMode") == "flat")
+        for (auto hashedMirror : settings.hashedMirrors.get())
+            try {
+                if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
+                auto ht = parseHashType(getAttr("outputHashAlgo"));
+                auto h = Hash(getAttr("outputHash"), ht);
+                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+                return;
+            } catch (Error & e) {
+                debug(e.what());
+            }
+
+    /* Otherwise try the specified URL. */
     fetch(mainUrl);
 }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3406a9331..e3bb4cf84 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -335,6 +335,9 @@ public:
         "setuid/setgid bits or with file capabilities."};
 #endif
 
+    Setting<Strings> hashedMirrors{this, {}, "hashed-mirrors",
+        "A list of servers used by builtins.fetchurl to fetch files by hash."};
+
     Setting<uint64_t> minFree{this, 0, "min-free",
         "Automatically run the garbage collector when free disk space drops below the specified amount."};
 

From 46f9dd56da7d2af82148c47e40108f3c11ffe4d7 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Thu, 6 Aug 2020 19:30:05 -0400
Subject: [PATCH 076/882] Fix bug due to non-deterministic arg eval order

---
 src/libstore/worker-protocol.hh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 60543d626..1e8fd027c 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -121,9 +121,9 @@ struct WorkerProto<std::map<K, V>> {
         std::map<K, V> resMap;
         auto size = readNum<size_t>(from);
         while (size--) {
-            resMap.insert_or_assign(
-                WorkerProto<K>::read(store, from),
-                WorkerProto<V>::read(store, from));
+            auto k = WorkerProto<K>::read(store, from);
+            auto v = WorkerProto<V>::read(store, from);
+            resMap.insert_or_assign(std::move(k), std::move(v));
         }
         return resMap;
     }

From 96c158d6e1c6c3bed6451a3c6447d2b0bd0337ad Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 6 Aug 2020 19:00:33 -0500
Subject: [PATCH 077/882] Fix build

---
 src/libstore/builtins/fetchurl.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 2048f8f87..4fb5d8a06 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -63,8 +63,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashType(getAttr("outputHashAlgo"));
-                auto h = Hash(getAttr("outputHash"), ht);
+                std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+                Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
                 fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {

From 2ffc0589507de63bea5555ba993ac195154b4517 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Aug 2020 14:13:24 +0200
Subject: [PATCH 078/882] Make --no-eval-cache a global setting

---
 src/libexpr/eval.hh     |  3 +++
 src/nix/app.cc          |  2 +-
 src/nix/flake.cc        | 11 ++---------
 src/nix/installables.cc | 22 +++++++++-------------
 src/nix/installables.hh |  9 ++++-----
 src/nix/search.cc       |  2 +-
 6 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 0382298b3..5855b4ef2 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -375,6 +375,9 @@ struct EvalSettings : Config
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
         "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+
+    Setting<bool> useEvalCache{this, true, "eval-cache",
+        "Whether to use the flake evaluation cache."};
 };
 
 extern EvalSettings evalSettings;
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 3935297cf..80acbf658 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -8,7 +8,7 @@ namespace nix {
 
 App Installable::toApp(EvalState & state)
 {
-    auto [cursor, attrPath] = getCursor(state, true);
+    auto [cursor, attrPath] = getCursor(state);
 
     auto type = cursor->getAttr("type")->getString();
 
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 80d8654bc..653f8db1b 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -572,7 +572,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             Strings{templateName == "" ? "defaultTemplate" : templateName},
             Strings(attrsPathPrefixes), lockFlags);
 
-        auto [cursor, attrPath] = installable.getCursor(*evalState, true);
+        auto [cursor, attrPath] = installable.getCursor(*evalState);
 
         auto templateDir = cursor->getAttr("path")->getString();
 
@@ -782,7 +782,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
 struct CmdFlakeShow : FlakeCommand
 {
     bool showLegacy = false;
-    bool useEvalCache = true;
 
     CmdFlakeShow()
     {
@@ -791,12 +790,6 @@ struct CmdFlakeShow : FlakeCommand
             .description = "show the contents of the 'legacyPackages' output",
             .handler = {&showLegacy, true}
         });
-
-        addFlag({
-            .longName = "no-eval-cache",
-            .description = "do not use the flake evaluation cache",
-            .handler = {[&]() { useEvalCache = false; }}
-        });
     }
 
     std::string description() override
@@ -934,7 +927,7 @@ struct CmdFlakeShow : FlakeCommand
             }
         };
 
-        auto cache = openEvalCache(*state, flake, useEvalCache);
+        auto cache = openEvalCache(*state, flake);
 
         visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
     }
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 06e61380b..ea152709f 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -183,8 +183,7 @@ void completeFlakeRefWithFragment(
             auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)),
-                true);
+                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
 
             auto root = evalCache->getRoot();
 
@@ -273,7 +272,7 @@ Buildable Installable::toBuildable()
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-Installable::getCursors(EvalState & state, bool useEvalCache)
+Installable::getCursors(EvalState & state)
 {
     auto evalCache =
         std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
@@ -282,9 +281,9 @@ Installable::getCursors(EvalState & state, bool useEvalCache)
 }
 
 std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
-Installable::getCursor(EvalState & state, bool useEvalCache)
+Installable::getCursor(EvalState & state)
 {
-    auto cursors = getCursors(state, useEvalCache);
+    auto cursors = getCursors(state);
     if (cursors.empty())
         throw Error("cannot find flake attribute '%s'", what());
     return cursors[0];
@@ -420,12 +419,11 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake,
-    bool useEvalCache)
+    std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
     auto fingerprint = lockedFlake->getFingerprint();
     return make_ref<nix::eval_cache::EvalCache>(
-        useEvalCache && evalSettings.pureEval
+        evalSettings.useEvalCache && evalSettings.pureEval
             ? std::optional { std::cref(fingerprint) }
             : std::nullopt,
         state,
@@ -460,10 +458,9 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
 
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
-
     auto lockedFlake = getLockedFlake();
 
-    auto cache = openEvalCache(*state, lockedFlake, true);
+    auto cache = openEvalCache(*state, lockedFlake);
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
@@ -517,11 +514,10 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-InstallableFlake::getCursors(EvalState & state, bool useEvalCache)
+InstallableFlake::getCursors(EvalState & state)
 {
     auto evalCache = openEvalCache(state,
-        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)),
-        useEvalCache);
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
 
     auto root = evalCache->getRoot();
 
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index 9edff3331..26e87ee3a 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -62,10 +62,10 @@ struct Installable
     }
 
     virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-    getCursors(EvalState & state, bool useEvalCache);
+    getCursors(EvalState & state);
 
     std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
-    getCursor(EvalState & state, bool useEvalCache);
+    getCursor(EvalState & state);
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
@@ -118,7 +118,7 @@ struct InstallableFlake : InstallableValue
     std::pair<Value *, Pos> toValue(EvalState & state) override;
 
     std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-    getCursors(EvalState & state, bool useEvalCache) override;
+    getCursors(EvalState & state) override;
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
@@ -127,7 +127,6 @@ struct InstallableFlake : InstallableValue
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake,
-    bool useEvalCache);
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
 
 }
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 65a1e1818..430979274 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -177,7 +177,7 @@ struct CmdSearch : InstallableCommand, MixJSON
             }
         };
 
-        for (auto & [cursor, prefix] : installable->getCursors(*state, true))
+        for (auto & [cursor, prefix] : installable->getCursors(*state))
             visit(*cursor, parseAttrPath(*state, prefix));
 
         if (!json && !results)

From 3c75ddc16b054a7dabf0710ee0a4323b4371effd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Aug 2020 14:46:00 +0200
Subject: [PATCH 079/882] nix build (and others): Force re-evaluation of cached
 errors

Fixes #3872.

This is a bit hacky. Ideally we would automatically re-evaluate the
failed attribute iff we need to print the error message (so in
commands like 'nix search' we wouldn't re-evaluate because we're
suppressing errors).
---
 src/libexpr/eval-cache.cc | 17 ++++++++++-------
 src/libexpr/eval-cache.hh |  6 ++++--
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index deb32484f..46177a0a4 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -405,7 +405,7 @@ Value & AttrCursor::forceValue()
     return v;
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
 {
     if (root->db) {
         if (!cachedValue)
@@ -422,9 +422,12 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second))
-                        throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
-                    else
+                    else if (std::get_if<failed_t>(&attr->second)) {
+                        if (forceErrors)
+                            debug("reevaluating failed cached attribute '%s'");
+                        else
+                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    } else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -469,9 +472,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
 {
-    auto p = maybeGetAttr(name);
+    auto p = maybeGetAttr(name, forceErrors);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return p;
@@ -600,7 +603,7 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath);
+    auto aDrvPath = getAttr(root->state.sDrvPath, true);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index afee85fa9..8ffffc0ed 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -9,6 +9,8 @@
 
 namespace nix::eval_cache {
 
+MakeError(CachedEvalError, EvalError);
+
 class AttrDb;
 class AttrCursor;
 
@@ -92,11 +94,11 @@ public:
 
     std::string getAttrPathStr(Symbol name) const;
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> getAttr(Symbol name);
+    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
 
     std::shared_ptr<AttrCursor> getAttr(std::string_view name);
 

From 47644e49ca252441fcc3ae57f5a01e7fc579ba8c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 7 Aug 2020 17:05:14 +0000
Subject: [PATCH 080/882] Specialize `std::optional<StorePath>` so this is
 backwards compatible

While I am cautious to break parametricity, I think it's OK in this
cases---we're not about to try to do some crazy polymorphic protocol
anytime soon.
---
 src/libstore/remote-store.cc    | 14 ++++++++++++++
 src/libstore/worker-protocol.hh |  7 +++++++
 2 files changed, 21 insertions(+)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 96e0aabba..005415666 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -70,6 +70,20 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
     out << store.printStorePath(storePath);
 }
 
+
+template<>
+std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
+{
+	auto s = readString(from);
+    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
+}
+
+template<>
+void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+{
+    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+}
+
 }
 
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index c50995d7c..b3576fbeb 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -90,6 +90,13 @@ std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<
 template<typename T>
 void write(const Store & store, Sink & out, const std::optional<T> & optVal);
 
+/* Specialization which uses and empty string for the empty case, taking
+   advantage of the fact StorePaths always serialize to a non-empty string.
+   This is done primarily for backwards compatability, so that StorePath <=
+   std::optional<StorePath>, where <= is the compatability partial order.
+ */
+template<>
+void write(const Store & store, Sink & out, const std::optional<StorePath> & optVal);
 
 template<typename T>
 std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _)

From edfd676e059578fb574ce78d1a2cc66d018d3b16 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Aug 2020 21:18:29 +0200
Subject: [PATCH 081/882] Fix .ls file names in binary caches

These are not supposed to include the 'name' part of the store
path. This was broken by 759947bf72.
---
 src/libstore/binary-cache-store.cc | 2 +-
 tests/binary-cache.sh              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 9682db730..86877dd1a 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -219,7 +219,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
             }
         }
 
-        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
     }
 
     /* Optionally maintain an index of DWARF debug info files
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index b05a7d167..fe4ddec8d 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -219,7 +219,7 @@ outPath=$(nix-build --no-out-link -E '
 nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
 
 diff -u \
-    <(jq -S < $cacheDir/$(basename $outPath).ls) \
+    <(jq -S < $cacheDir/$(basename $outPath | cut -c1-32).ls) \
     <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S)
 
 

From e913a2989fd7dfabfd93c89fd4295386eda4277f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 7 Aug 2020 19:09:26 +0000
Subject: [PATCH 082/882] Squashed get CA derivations building

---
 src/libexpr/get-drvs.cc      |  13 +-
 src/libexpr/get-drvs.hh      |   2 +-
 src/libexpr/primops.cc       |  59 ++-
 src/libstore/build.cc        | 964 ++++++++++++++++++++++++-----------
 src/libstore/derivations.cc  | 158 +++---
 src/libstore/derivations.hh  |  14 +-
 src/libstore/local-store.cc  |  31 +-
 src/libstore/local-store.hh  |   6 +
 src/libstore/misc.cc         |  23 +-
 src/libstore/references.cc   |  16 +-
 src/libstore/references.hh   |   2 +
 src/libstore/remote-store.cc |   2 +-
 src/libstore/store-api.cc    |  19 +-
 src/libstore/store-api.hh    |   6 +-
 src/libutil/serialise.hh     |   6 +
 src/nix-build/nix-build.cc   |  56 +-
 src/nix-store/nix-store.cc   |  26 +-
 src/nix/build.cc             |   3 +-
 src/nix/command.cc           |   4 +-
 src/nix/installables.cc      |  13 +-
 src/nix/installables.hh      |   4 +-
 src/nix/profile.cc           |  10 +-
 src/nix/repl.cc              |   6 +-
 src/nix/show-derivation.cc   |   4 +-
 tests/content-addressed.nix  |  19 +
 tests/content-addressed.sh   |  16 +
 tests/local.mk               |   3 +-
 tests/multiple-outputs.nix   |  15 +
 tests/multiple-outputs.sh    |   6 +
 29 files changed, 1021 insertions(+), 485 deletions(-)
 create mode 100644 tests/content-addressed.nix
 create mode 100644 tests/content-addressed.sh

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 5d6e39aa0..d41cac132 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -39,7 +39,9 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
     if (i == drv.outputs.end())
         throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
 
-    outPath = store->printStorePath(i->second.path(*store, drv.name));
+    auto optStorePath = i->second.pathOpt(*store, drv.name);
+    if (optStorePath)
+        outPath = store->printStorePath(*optStorePath);
 }
 
 
@@ -77,12 +79,15 @@ string DrvInfo::queryDrvPath() const
 
 string DrvInfo::queryOutPath() const
 {
-    if (outPath == "" && attrs) {
+    if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
-        outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
+        if (i != attrs->end())
+            outPath = state->coerceToPath(*i->pos, *i->value, context);
     }
-    return outPath;
+    if (!outPath)
+        throw UnimplementedError("CA derivations are not yet supported");
+    return *outPath;
 }
 
 
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index d7860fc6a..29bb6a660 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -20,7 +20,7 @@ private:
     mutable string name;
     mutable string system;
     mutable string drvPath;
-    mutable string outPath;
+    mutable std::optional<string> outPath;
     mutable string outputName;
     Outputs outputs;
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 65d36ca0e..11f44fb97 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -44,16 +44,6 @@ void EvalState::realiseContext(const PathSet & context)
             throw InvalidPathError(store->printStorePath(ctx));
         if (!outputName.empty() && ctx.isDerivation()) {
             drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
-
-            /* Add the output of this derivation to the allowed
-               paths. */
-            if (allowedPaths) {
-                auto drv = store->derivationFromPath(ctx);
-                DerivationOutputs::iterator i = drv.outputs.find(outputName);
-                if (i == drv.outputs.end())
-                    throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
-                allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
-            }
         }
     }
 
@@ -69,8 +59,38 @@ void EvalState::realiseContext(const PathSet & context)
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     store->buildPaths(drvs);
+
+    /* Add the output of this derivations to the allowed
+       paths. */
+    if (allowedPaths) {
+        for (auto & [drvPath, outputs] : drvs) {
+            auto outputPaths = store->queryDerivationOutputMapAssumeTotal(drvPath);
+            for (auto & outputName : outputs) {
+                if (outputPaths.count(outputName) == 0)
+                    throw Error("derivation '%s' does not have an output named '%s'",
+                            store->printStorePath(drvPath), outputName);
+                allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
+            }
+        }
+    }
 }
 
+static void mkOutputString(EvalState & state, Value & v,
+    const StorePath & drvPath, const BasicDerivation & drv,
+    std::pair<string, DerivationOutput> o)
+{
+    auto optOutputPath = o.second.pathOpt(*state.store, drv.name);
+    mkString(
+        *state.allocAttr(v, state.symbols.create(o.first)),
+        state.store->printStorePath(optOutputPath
+            ? *optOutputPath
+            /* Downstream we would substitute this for an actual path once
+               we build the floating CA derivation */
+            /* FIXME: we need to depend on the basic derivation, not
+               derivation */
+            : downstreamPlaceholder(*state.store, drvPath, o.first)),
+        {"!" + o.first + "!" + state.store->printStorePath(drvPath)});
+}
 
 /* Load and evaluate an expression from path specified by the
    argument. */
@@ -114,8 +134,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         unsigned int outputs_index = 0;
 
         for (const auto & o : drv.outputs) {
-            v2 = state.allocAttr(w, state.symbols.create(o.first));
-            mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
+            mkOutputString(state, w, storePath, drv, o);
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
@@ -846,16 +865,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
-       read them later. */
-    drvHashes.insert_or_assign(drvPath,
-        hashDerivationModulo(*state.store, Derivation(drv), false));
+       read them later.
+
+       However, we don't bother doing this for floating CA derivations because
+       their "hash modulo" is indeterminate until built. */
+    if (drv.type() != DerivationType::CAFloating)
+        drvHashes.insert_or_assign(drvPath,
+            hashDerivationModulo(*state.store, Derivation(drv), false));
 
     state.mkAttrs(v, 1 + drv.outputs.size());
     mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
-    for (auto & i : drv.outputs) {
-        mkString(*state.allocAttr(v, state.symbols.create(i.first)),
-            state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
-    }
+    for (auto & i : drv.outputs)
+        mkOutputString(state, v, drvPath, drv, i);
     v.attrs->sort();
 }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b47aeff3b..7c2225805 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -16,6 +16,7 @@
 #include "machines.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
+#include "topo-sort.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -717,6 +718,24 @@ typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
 
+struct KnownInitialOutputStatus {
+    StorePath path;
+    /* The output optional indicates whether it's already valid; i.e. exists
+       and is registered. If we're repairing, inner bool indicates whether the
+       valid path is in fact not corrupt. Otherwise, the inner bool is always
+       true (assumed no corruption). */
+    std::optional<bool> valid;
+    /* Valid in the store, and additionally non-corrupt if we are repairing */
+    bool isValid() const {
+        return valid && *valid;
+    }
+};
+
+struct InitialOutputStatus {
+    bool wanted;
+    std::optional<KnownInitialOutputStatus> known;
+};
+
 class DerivationGoal : public Goal
 {
 private:
@@ -744,19 +763,14 @@ private:
 
     /* The remainder is state held during the build. */
 
-    /* Locks on the output paths. */
+    /* Locks on (fixed) output paths. */
     PathLocks outputLocks;
 
     /* All input paths (that is, the union of FS closures of the
        immediate input paths). */
     StorePathSet inputPaths;
 
-    /* Outputs that are already valid.  If we're repairing, these are
-       the outputs that are valid *and* not corrupt. */
-    StorePathSet validPaths;
-
-    /* Outputs that are corrupt or not valid. */
-    StorePathSet missingPaths;
+    std::map<std::string, InitialOutputStatus> initialOutputs;
 
     /* User selected for running the builder. */
     std::unique_ptr<UserLock> buildUser;
@@ -839,6 +853,31 @@ private:
     typedef map<StorePath, StorePath> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
+    /* The outputs paths used during the build.
+
+       - Input-addressed derivations or fixed content-addressed outputs are
+         sometimes built when some of their outputs already exist, and can not
+         be hidden via sandboxing. We use temporary locations instead and
+         rewrite after the build. Otherwise the regular predetermined paths are
+         put here.
+
+       - Floating content-addressed derivations do not know their final build
+         output paths until the outputs are hashed, so random locations are
+         used, and then renamed. The randomness helps guard against hidden
+         self-references.
+     */
+    OutputPathMap scratchOutputs;
+
+    /* The final output paths of the build.
+
+       - For input-addressed derivations, always the precomputed paths
+
+       - For content-addressed derivations, calcuated from whatever the hash
+         ends up being. (Note that fixed outputs derivations that produce the
+         "wrong" output still install that data under its true content-address.)
+     */
+    std::map<std::string, StorePath> finalOutputs;
+
     BuildMode buildMode;
 
     /* If we're repairing without a chroot, there may be outputs that
@@ -937,7 +976,8 @@ private:
     void getDerivation();
     void loadDerivation();
     void haveDerivation();
-    void outputsSubstituted();
+    void outputsSubstitutionTried();
+    void gaveUpOnSubstitution();
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
@@ -998,13 +1038,24 @@ private:
     void handleEOF(int fd) override;
     void flushLine();
 
+    /* Wrappers around the corresponding Store methods that first consult the
+       derivation.  This is currently needed because when there is no drv file
+       there also is no DB entry. */
+    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap();
+    OutputPathMap queryDerivationOutputMapAssumeTotal();
+
     /* Return the set of (in)valid paths. */
-    StorePathSet checkPathValidity(bool returnValid, bool checkHash);
+    void checkPathValidity();
 
     /* Forcibly kill the child process, if any. */
     void killChild();
 
-    void addHashRewrite(const StorePath & path);
+    /* Map a path to another (reproducably) so we can avoid overwriting outputs
+       that already exist. */
+    StorePath makeFallbackPath(const StorePath & path);
+    /* Make a path to another based on the output name alone, if one doesn't
+       want to use a random path for CA builds. */
+    StorePath makeFallbackPath(std::string_view path);
 
     void repairClosure();
 
@@ -1047,7 +1098,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
 {
     this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
     state = &DerivationGoal::haveDerivation;
-    name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store)));
+    name = fmt("building of %s", StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1179,43 +1230,63 @@ void DerivationGoal::haveDerivation()
 {
     trace("have derivation");
 
+    if (drv->type() == DerivationType::CAFloating)
+        settings.requireExperimentalFeature("ca-derivations");
+
     retrySubstitution = false;
 
-    for (auto & i : drv->outputs)
-        worker.store.addTempRoot(i.second.path(worker.store, drv->name));
+    /* Temporarily root output paths that are known a priori building */
+    for (auto & i : drv->outputs) {
+        auto optOutputPath = i.second.pathOpt(worker.store, drv->name);
+        if (optOutputPath)
+            worker.store.addTempRoot(*optOutputPath);
+    }
 
     /* Check what outputs paths are not already valid. */
-    auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
+    checkPathValidity();
+    bool allValid = true;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid()) {
+            allValid = false;
+            break;
+        }
+    }
 
     /* If they are all valid, then we're done. */
-    if (invalidOutputs.size() == 0 && buildMode == bmNormal) {
+    if (allValid && buildMode == bmNormal) {
         done(BuildResult::AlreadyValid);
         return;
     }
 
     parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
 
-    if (drv->type() == DerivationType::CAFloating) {
-        settings.requireExperimentalFeature("ca-derivations");
-        throw UnimplementedError("ca-derivations isn't implemented yet");
-    }
-
 
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
     if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & i : invalidOutputs)
-            addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
+        for (auto & [_, status] : initialOutputs) {
+            if (!status.wanted) continue;
+            if (!status.known) {
+                warn("Do not know how to query for unknown floating CA drv output yet");
+                /* Nothing to wait for; tail call */
+                return DerivationGoal::gaveUpOnSubstitution();
+            }
+            addWaitee(worker.makeSubstitutionGoal(
+                status.known->path,
+                buildMode == bmRepair ? Repair : NoRepair,
+                getDerivationCA(*drv)));
+        }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstituted();
+        outputsSubstitutionTried();
     else
-        state = &DerivationGoal::outputsSubstituted;
+        state = &DerivationGoal::outputsSubstitutionTried;
 }
 
 
-void DerivationGoal::outputsSubstituted()
+void DerivationGoal::outputsSubstitutionTried()
 {
     trace("all outputs substituted (maybe)");
 
@@ -1239,7 +1310,14 @@ void DerivationGoal::outputsSubstituted()
         return;
     }
 
-    auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size();
+    checkPathValidity();
+    size_t nrInvalid = 0;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid())
+            nrInvalid++;
+    }
+
     if (buildMode == bmNormal && nrInvalid == 0) {
         done(BuildResult::Substituted);
         return;
@@ -1252,6 +1330,12 @@ void DerivationGoal::outputsSubstituted()
         throw Error("some outputs of '%s' are not valid, so checking is not possible",
             worker.store.printStorePath(drvPath));
 
+    /* Nothing to wait for; tail call */
+    gaveUpOnSubstitution();
+}
+
+void DerivationGoal::gaveUpOnSubstitution()
+{
     /* Otherwise, at least one of the output paths could not be
        produced using a substitute.  So we have to build instead. */
 
@@ -1287,15 +1371,16 @@ void DerivationGoal::repairClosure()
        that produced those outputs. */
 
     /* Get the output closure. */
+    auto outputs = queryDerivationOutputMapAssumeTotal();
     StorePathSet outputClosure;
-    for (auto & i : drv->outputs) {
+    for (auto & i : outputs) {
         if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
+        worker.store.computeFSClosure(i.second, outputClosure);
     }
 
     /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : drv->outputs)
-        outputClosure.erase(i.second.path(worker.store, drv->name));
+    for (auto & i : outputs)
+        outputClosure.erase(i.second);
 
     /* Get all dependencies of this derivation so that we know which
        derivation is responsible for which path in the output
@@ -1305,9 +1390,9 @@ void DerivationGoal::repairClosure()
     std::map<StorePath, StorePath> outputsToDrv;
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(i);
-            for (auto & j : drv.outputs)
-                outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
+            auto depOutputs = worker.store.queryDerivationOutputMapAssumeTotal(i);
+            for (auto & j : depOutputs)
+                outputsToDrv.insert_or_assign(j.second, i);
         }
 
     /* Check each path (slow!). */
@@ -1370,20 +1455,19 @@ void DerivationGoal::inputsRealised()
 
     /* First, the input derivations. */
     if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+        for (auto & [depDrvPath, wantedDepOutputs] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            assert(worker.store.isValidPath(i.first));
-            Derivation inDrv = worker.store.derivationFromPath(i.first);
-            for (auto & j : i.second) {
-                auto k = inDrv.outputs.find(j);
-                if (k != inDrv.outputs.end())
-                    worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths);
+            assert(worker.store.isValidPath(drvPath));
+            auto outputs = worker.store.queryDerivationOutputMapAssumeTotal(depDrvPath);
+            for (auto & j : wantedDepOutputs) {
+                if (outputs.count(j) > 0)
+                    worker.store.computeFSClosure(outputs.at(j), inputPaths);
                 else
                     throw Error(
                         "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first));
+                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
             }
         }
 
@@ -1426,14 +1510,20 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
-    /* Obtain locks on all output paths.  The locks are automatically
-       released when we exit this function or Nix crashes.  If we
-       can't acquire the lock, then continue; hopefully some other
-       goal can start a build, and if not, the main loop will sleep a
-       few seconds and then retry this goal. */
+    /* Obtain locks on all output paths, if the paths are known a priori.
+
+       The locks are automatically released when we exit this function or Nix
+       crashes.  If we can't acquire the lock, then continue; hopefully some
+       other goal can start a build, and if not, the main loop will sleep a few
+       seconds and then retry this goal. */
     PathSet lockFiles;
-    for (auto & outPath : drv->outputPaths(worker.store))
-        lockFiles.insert(worker.store.Store::toRealPath(outPath));
+    /* FIXME: Should lock something like the drv itself so we don't build same
+       CA drv concurrently */
+    for (auto & i : drv->outputs) {
+        auto optPath = i.second.pathOpt(worker.store, drv->name);
+        if (optPath)
+            lockFiles.insert(worker.store.Store::toRealPath(*optPath));
+    }
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         if (!actLock)
@@ -1452,24 +1542,28 @@ void DerivationGoal::tryToBuild()
        omitted, but that would be less efficient.)  Note that since we
        now hold the locks on the output paths, no other process can
        build this derivation, so no further checks are necessary. */
-    validPaths = checkPathValidity(true, buildMode == bmRepair);
-    if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) {
+    bool allValid = true;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid()) {
+            allValid = false;
+            break;
+        }
+    }
+    if (buildMode != bmCheck && allValid) {
         debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
         outputLocks.setDeletion(true);
         done(BuildResult::AlreadyValid);
         return;
     }
 
-    missingPaths = drv->outputPaths(worker.store);
-    if (buildMode != bmCheck)
-        for (auto & i : validPaths) missingPaths.erase(i);
-
     /* If any of the outputs already exist but are not valid, delete
        them. */
-    for (auto & i : drv->outputs) {
-        if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
-        debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
-        deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.known || status.known->isValid()) continue;
+        auto storePath = status.known->path;
+        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
+        deletePath(worker.store.Store::toRealPath(storePath));
     }
 
     /* Don't do a remote build if the derivation has the attribute
@@ -1477,7 +1571,6 @@ void DerivationGoal::tryToBuild()
        supported for local builds. */
     bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
 
-    /* Is the build hook willing to accept this job? */
     if (!buildLocally) {
         switch (tryBuildHook()) {
             case rpAccept:
@@ -1661,8 +1754,10 @@ void DerivationGoal::buildDone()
             /* Move paths out of the chroot for easier debugging of
                build failures. */
             if (useChroot && buildMode == bmNormal)
-                for (auto & i : missingPaths) {
-                    auto p = worker.store.printStorePath(i);
+                for (auto & [_, status] : initialOutputs) {
+                    if (!status.known) continue;
+                    if (buildMode != bmCheck && status.known->isValid()) continue;
+                    auto p = worker.store.printStorePath(status.known->path);
                     if (pathExists(chrootRootDir + p))
                         rename((chrootRootDir + p).c_str(), p.c_str());
                 }
@@ -1692,7 +1787,10 @@ void DerivationGoal::buildDone()
                 fmt("running post-build-hook '%s'", settings.postBuildHook),
                 Logger::Fields{worker.store.printStorePath(drvPath)});
             PushActivity pact(act.id);
-            auto outputPaths = drv->outputPaths(worker.store);
+            StorePathSet outputPaths;
+            for (auto i : drv->outputs) {
+                outputPaths.insert(finalOutputs.at(i.first));
+            }
             std::map<std::string, std::string> hookEnvironment = getEnv();
 
             hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
@@ -1868,7 +1966,15 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
-    writeStorePaths(worker.store, hook->sink, missingPaths);
+    {
+        StorePathSet missingPaths;
+        for (auto & [_, status] : initialOutputs) {
+            if (!status.known) continue;
+            if (buildMode != bmCheck && status.known->isValid()) continue;
+            missingPaths.insert(status.known->path);
+        }
+        writeStorePaths(worker.store, hook->sink, missingPaths);
+    }
 
     hook->sink = FdSink();
     hook->toHook.writeSide = -1;
@@ -1919,8 +2025,16 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
     for (auto & j : paths2) {
         if (j.isDerivation()) {
             Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputs)
-                worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
+            for (auto & k : drv.outputs) {
+                auto optPath = k.second.pathOpt(worker.store, drv.name);
+                if (!optPath)
+                    /* FIXME: I am confused why we are calling
+                       `computeFSClosure` on the output path, rather than
+                       derivation itself. That doesn't seem right to me, so I
+                       won't try to implemented this for CA derivations. */
+                    throw UnimplementedError("export references including CA derivations (themselves) is not yet implemented");
+                worker.store.computeFSClosure(*optPath, paths);
+            }
         }
     }
 
@@ -2013,9 +2127,66 @@ void DerivationGoal::startBuilder()
 
     chownToBuilder(tmpDir);
 
-    /* Substitute output placeholders with the actual output paths. */
-    for (auto & output : drv->outputs)
-        inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
+    for (auto & [outputName, status] : initialOutputs) {
+        /* Set scratch path we'll actually use during the build.
+
+           If we're not doing a chroot build, but we have some valid
+           output paths.  Since we can't just overwrite or delete
+           them, we have to do hash rewriting: i.e. in the
+           environment/arguments passed to the build, we replace the
+           hashes of the valid outputs with unique dummy strings;
+           after the build, we discard the redirected outputs
+           corresponding to the valid outputs, and rewrite the
+           contents of the new outputs to replace the dummy strings
+           with the actual hashes. */
+        auto scratchPath =
+            !status.known
+                /* FIXME add option to randomize, so we can audit whether our
+                 * rewrites caught everything */
+                ? makeFallbackPath(outputName)
+            : !needsHashRewrite()
+                /* Can always use original path in sandbox */
+                ? status.known->path
+            : !status.known->valid
+                /* If path doesn't yet exist can just use it */
+                ? status.known->path
+            : buildMode != bmRepair && !*status.known->valid
+                /* If we aren't repairing we'll delete a corrupted path, so we
+                   can use original path */
+                ? status.known->path
+            :   /* If we are repairing or the path is totally valid, we'll need
+                   to use a temporary path */
+                makeFallbackPath(status.known->path);
+        scratchOutputs.insert_or_assign(outputName, scratchPath);
+
+        /* A non-removed corrupted path needs to be stored here, too */
+        if (buildMode == bmRepair && !*status.known->valid)
+            redirectedBadOutputs.insert(status.known->path);
+
+        /* Substitute output placeholders with the scratch output paths.
+           We'll use during the build. */
+        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
+
+        /* Additional tasks if we know the final path a priori. */
+        if (!status.known) continue;
+        auto fixedFinalPath = status.known->path;
+
+        /* Additional tasks if the final and scratch are both known and
+           differ. */
+        if (fixedFinalPath == scratchPath) continue;
+
+        /* Ensure scratch scratch path is ours to use */
+        deletePath(worker.store.printStorePath(scratchPath));
+
+        /* Rewrite and unrewrite paths */
+        {
+            std::string h1 { fixedFinalPath.hashPart() };
+            std::string h2 { scratchPath.hashPart() };
+            inputRewrites[h1] = h2;
+        }
+
+        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
+    }
 
     /* Construct the environment passed to the builder. */
     initEnv();
@@ -2199,8 +2370,14 @@ void DerivationGoal::startBuilder()
            rebuilding a path that is in settings.dirsInChroot
            (typically the dependencies of /bin/sh).  Throw them
            out. */
-        for (auto & i : drv->outputs)
-            dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
+        for (auto & i : drv->outputs) {
+            /* If the name isn't known a prior (i.e. floating content-addressed
+               derivation), the temporary location we use should be fresh and
+               never in the sandbox in the first place. */
+            auto optPath = i.second.pathOpt(worker.store, drv->name);
+            if (optPath)
+                dirsInChroot.erase(worker.store.printStorePath(*optPath));
+        }
 
 #elif __APPLE__
         /* We don't really have any parent prep work to do (yet?)
@@ -2210,33 +2387,8 @@ void DerivationGoal::startBuilder()
 #endif
     }
 
-    if (needsHashRewrite()) {
-
-        if (pathExists(homeDir))
-            throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-        /* We're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        if (validPaths.size() > 0)
-            for (auto & i : validPaths)
-                addHashRewrite(i);
-
-        /* If we're repairing, then we don't want to delete the
-           corrupt outputs in advance.  So rewrite them as well. */
-        if (buildMode == bmRepair)
-            for (auto & i : missingPaths)
-                if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) {
-                    addHashRewrite(i);
-                    redirectedBadOutputs.insert(i);
-                }
-    }
+    if (needsHashRewrite() && pathExists(homeDir))
+        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
 
     if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
         printMsg(lvlChatty, format("executing pre-build hook '%1%'")
@@ -2612,8 +2764,11 @@ void DerivationGoal::writeStructuredAttrs()
 
     /* Add an "outputs" object containing the output paths. */
     nlohmann::json outputs;
-    for (auto & i : drv->outputs)
-        outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
+    for (auto & i : drv->outputs) {
+        /* The placeholder must have a rewrite, so we use it to cover both the
+           cases where we know or don't know the output path ahead of time. */
+        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
+    }
     json["outputs"] = outputs;
 
     /* Handle exportReferencesGraph. */
@@ -2815,19 +2970,20 @@ struct RestrictedStore : public LocalFSStore
         StorePathSet newPaths;
 
         for (auto & path : paths) {
-            if (path.path.isDerivation()) {
-                if (!goal.isAllowed(path.path))
-                    throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-                auto drv = derivationFromPath(path.path);
-                for (auto & output : drv.outputs)
-                    if (wantOutput(output.first, path.outputs))
-                        newPaths.insert(output.second.path(*this, drv.name));
-            } else if (!goal.isAllowed(path.path))
+            if (!goal.isAllowed(path.path))
                 throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
         }
 
         next->buildPaths(paths, buildMode);
 
+        for (auto & path : paths) {
+            if (!path.path.isDerivation()) continue;
+            auto outputs = next->queryDerivationOutputMapAssumeTotal(path.path);
+            for (auto & output : outputs)
+                if (wantOutput(output.first, path.outputs))
+                    newPaths.insert(output.second);
+        }
+
         StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
@@ -3443,14 +3599,10 @@ void DerivationGoal::runChild()
                 if (derivationIsImpure(derivationType))
                     sandboxProfile += "(import \"sandbox-network.sb\")\n";
 
-                /* Our rwx outputs */
+                /* Add the output paths we'll use at build-time to the chroot */
                 sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : missingPaths)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i));
-
-                /* Also add redirected outputs to the chroot */
-                for (auto & i : redirectedOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second));
+                for (auto & [_, path] : scratchOutputs)
+                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
 
                 sandboxProfile += ")\n";
 
@@ -3573,23 +3725,6 @@ void DerivationGoal::runChild()
 }
 
 
-/* Parse a list of reference specifiers.  Each element must either be
-   a store path, or the symbolic name of the output of the derivation
-   (such as `out'). */
-StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths)
-{
-    StorePathSet result;
-    for (auto & i : paths) {
-        if (store.isStorePath(i))
-            result.insert(store.parseStorePath(i));
-        else if (drv.outputs.count(i))
-            result.insert(drv.outputs.find(i)->second.path(store, drv.name));
-        else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-    }
-    return result;
-}
-
-
 static void moveCheckToStore(const Path & src, const Path & dst)
 {
     /* For the rename of directory to succeed, we must be running as root or
@@ -3617,11 +3752,18 @@ void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
        as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here. */
+       to do anything here.
+
+       We can only early return when the outputs are known a priori. For
+       floating content-addressed derivations this isn't the case.
+     */
     if (hook) {
         bool allValid = true;
-        for (auto & i : drv->outputs)
-            if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
+        for (auto & i : drv->outputs) {
+            auto optStorePath = i.second.pathOpt(worker.store, drv->name);
+            if (!optStorePath || !worker.store.isValidPath(*optStorePath))
+                allValid = false;
+        }
         if (allValid) return;
     }
 
@@ -3642,47 +3784,47 @@ void DerivationGoal::registerOutputs()
        Nix calls. */
     StorePathSet referenceablePaths;
     for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
+    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
     for (auto & p : addedPaths) referenceablePaths.insert(p);
 
-    /* Check whether the output paths were created, and grep each
-       output path to determine what other paths it references.  Also make all
-       output paths read-only. */
-    for (auto & i : drv->outputs) {
-        auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
-        if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
+    /* FIXME `needsHashRewrite` should probably be removed and we get to the
+       real reason why we aren't using the chroot dir */
+    auto toRealPathChroot = [&](const Path & p) -> Path {
+        return useChroot && !needsHashRewrite()
+            ? chrootRootDir + p
+            : worker.store.toRealPath(p);
+    };
 
-        Path actualPath = path;
-        if (needsHashRewrite()) {
-            auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
-            if (r != redirectedOutputs.end()) {
-                auto redirected = worker.store.Store::toRealPath(r->second);
-                if (buildMode == bmRepair
-                    && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
-                    && pathExists(redirected))
-                    replaceValidPath(path, redirected);
-                if (buildMode == bmCheck)
-                    actualPath = redirected;
-            }
-        } else if (useChroot) {
-            actualPath = chrootRootDir + path;
-            if (pathExists(actualPath)) {
-                /* Move output paths from the chroot to the Nix store. */
-                if (buildMode == bmRepair)
-                    replaceValidPath(path, actualPath);
-                else
-                    if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1)
-                        throw SysError("moving build output '%1%' from the sandbox to the Nix store", path);
-            }
-            if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
+    /* Check whether the output paths were created, and make all
+       output paths read-only.  Then get the references of each output (that we
+       might need to register), so we can topologically sort them. For the ones
+       that are most definitely already installed, we just store their final
+       name so we can also use it in rewrites. */
+    StringSet outputsToSort;
+    std::map<std::string, std::variant<StorePath, StorePathSet>> outputReferences;
+    std::map<std::string, struct stat> outputStats;
+    for (auto & [outputName, _] : drv->outputs) {
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
+
+        outputsToSort.insert(outputName);
+
+        /* Updated wanted info to remove the outputs we definitely don't need to register */
+        auto & initialInfo = initialOutputs.at(outputName);
+
+        /* Don't register if already valid, and not checking */
+        initialInfo.wanted = buildMode == bmCheck
+            || !(initialInfo.known && initialInfo.known->isValid());
+        if (!initialInfo.wanted) {
+            outputReferences.insert_or_assign(outputName, initialInfo.known->path);
+            continue;
         }
 
         struct stat st;
         if (lstat(actualPath.c_str(), &st) == -1) {
             if (errno == ENOENT)
                 throw BuildError(
-                    "builder for '%s' failed to produce output path '%s'",
-                    worker.store.printStorePath(drvPath), path);
+                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
+                    worker.store.printStorePath(drvPath), outputName, actualPath);
             throw SysError("getting attributes of path '%s'", actualPath);
         }
 
@@ -3693,116 +3835,280 @@ void DerivationGoal::registerOutputs()
            user. */
         if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
             (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path);
+            throw BuildError(
+                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
+                    actualPath, outputName);
 #endif
 
-        /* Apply hash rewriting if necessary. */
+        /* Canonicalise first.  This ensures that the path we're
+           rewriting doesn't contain a hard link to /etc/shadow or
+           something like that. */
+        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
+
+        debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath);
+
+        /* Pass blank Sink as we are not ready to hash data at this stage. */
+        NullSink blank;
+        auto references = worker.store.parseStorePathSet(
+            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
+
+        outputReferences.insert_or_assign(outputName, references);
+        outputStats.insert_or_assign(outputName, std::move(st));
+    }
+
+    auto sortedOutputNames = topoSort(outputsToSort,
+        {[&](const std::string & name) {
+            auto x = outputReferences.at(name);
+            return std::visit(overloaded {
+                /* Since we'll use the already installed versions of these, we
+                   can treat them as leaves and ignore any references they
+                   have. */
+                [&](StorePath _) { return StringSet {}; },
+                [&](StorePathSet refs) {
+                    StringSet referencedOutputs;
+                    /* FIXME build inverted map up front so no quadratic waste here */
+                    for (auto & r : refs)
+                        for (auto & [o, p] : scratchOutputs)
+                            if (r == p)
+                                referencedOutputs.insert(o);
+                    return referencedOutputs;
+                },
+            }, x);
+        }},
+        {[&](const std::string & path, const std::string & parent) {
+            // TODO with more -vvvv also show the temporary paths for manual inspection.
+            return BuildError(
+                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
+                worker.store.printStorePath(drvPath), path, parent);
+        }});
+
+    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
+
+    for (auto & outputName : sortedOutputNames) {
+        auto output = drv->outputs.at(outputName);
+        auto & scratchPath = scratchOutputs.at(outputName);
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
+
+        auto finish = [&](StorePath finalStorePath) {
+            /* Store the final path */
+            finalOutputs.insert_or_assign(outputName, finalStorePath);
+            /* The rewrite rule will be used in downstream outputs that refer to
+               use. This is why the topological sort is essential to do first
+               before this for loop. */
+            if (scratchPath != finalStorePath)
+                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
+        };
+
         bool rewritten = false;
-        if (!outputRewrites.empty()) {
-            logWarning({
-                .name = "Rewriting hashes",
-                .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
-            });
+        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
+            [&](StorePath skippedFinalPath) -> std::optional<StorePathSet> {
+                finish(skippedFinalPath);
+                return std::nullopt;
+            },
+            [&](StorePathSet references) -> std::optional<StorePathSet> {
+                return references;
+            },
+        }, outputReferences.at(outputName));
 
-            /* Canonicalise first.  This ensures that the path we're
-               rewriting doesn't contain a hard link to /etc/shadow or
-               something like that. */
-            canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
+        if (!referencesOpt)
+            continue;
+        auto references = *referencesOpt;
 
-            /* FIXME: this is in-memory. */
-            StringSink sink;
-            dumpPath(actualPath, sink);
-            deletePath(actualPath);
-            sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-            StringSource source(*sink.s);
-            restorePath(actualPath, source);
+        auto rewriteOutput = [&]() {
+            /* Apply hash rewriting if necessary. */
+            if (!outputRewrites.empty()) {
+                logWarning({
+                    .name = "Rewriting hashes",
+                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
+                });
 
-            rewritten = true;
-        }
+                /* FIXME: this is in-memory. */
+                StringSink sink;
+                dumpPath(actualPath, sink);
+                deletePath(actualPath);
+                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
+                StringSource source(*sink.s);
+                restorePath(actualPath, source);
 
-        /* Check that fixed-output derivations produced the right
-           outputs (i.e., the content hash should match the specified
-           hash). */
-        std::optional<ContentAddress> ca;
+                rewritten = true;
+            }
+        };
 
-        if (! std::holds_alternative<DerivationOutputInputAddressed>(i.second.output)) {
-            DerivationOutputCAFloating outputHash;
-            std::visit(overloaded {
-                [&](DerivationOutputInputAddressed doi) {
-                    assert(false); // Enclosing `if` handles this case in other branch
+        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
+            /* In the CA case, we need the rewritten refs to calculate the
+               final path, therefore we look for a *non-rewritten
+               self-reference, and use a bool rather try to solve the
+               computationally intractable fixed point. */
+            std::pair<bool, StorePathSet> res {
+                false,
+                {},
+            };
+            for (auto & r : references) {
+                auto name = r.name();
+                auto origHash = std::string { r.hashPart() };
+                if (r == scratchPath)
+                    res.first = true;
+                else if (outputRewrites.count(origHash) == 0)
+                    res.second.insert(r);
+                else {
+                    std::string newRef = outputRewrites.at(origHash);
+                    newRef += '-';
+                    newRef += name;
+                    res.second.insert(StorePath { newRef });
+                }
+            }
+            return res;
+        };
+
+        ValidPathInfo newInfo = [&]() {if (auto outputP = std::get_if<DerivationOutputInputAddressed>(&output.output)) {
+            /* input-addressed case */
+            auto requiredFinalPath = outputP->path;
+            /* Preemtively add rewrite rule for final hash, as that is
+               what the NAR hash will use rather than normalized-self references */
+            if (scratchPath != requiredFinalPath)
+                outputRewrites.insert_or_assign(
+                    std::string { scratchPath.hashPart() },
+                    std::string { requiredFinalPath.hashPart() });
+            rewriteOutput();
+            auto narHashAndSize = hashPath(htSHA256, actualPath);
+            ValidPathInfo newInfo0 { requiredFinalPath };
+            newInfo0.narHash = narHashAndSize.first;
+            newInfo0.narSize = narHashAndSize.second;
+            auto refs = rewriteRefs();
+            newInfo0.references = refs.second;
+            if (refs.first)
+                newInfo0.references.insert(newInfo0.path);
+            return newInfo0;
+        } else {
+            /* content-addressed case */
+            DerivationOutputCAFloating outputHash = std::visit(overloaded {
+                [&](DerivationOutputInputAddressed doi) -> DerivationOutputCAFloating {
+                    // Enclosing `if` handles this case in other branch
+                    throw Error("ought to unreachable, handled in other branch");
                 },
                 [&](DerivationOutputCAFixed dof) {
-                    outputHash = DerivationOutputCAFloating {
+                    return DerivationOutputCAFloating {
                         .method = dof.hash.method,
                         .hashType = dof.hash.hash.type,
                     };
                 },
                 [&](DerivationOutputCAFloating dof) {
-                    outputHash = dof;
+                   return dof;
                 },
-            }, i.second.output);
-
+            }, output.output);
+            auto & st = outputStats.at(outputName);
             if (outputHash.method == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
                         "output path '%1%' should be a non-executable regular file "
                         "since recursive hashing is not enabled (outputHashMode=flat)",
-                        path);
+                        actualPath);
             }
-
-            /* Check the hash. In hash mode, move the path produced by
-               the derivation to its content-addressed location. */
-            Hash h2 = outputHash.method == FileIngestionMethod::Recursive
-                ? hashPath(outputHash.hashType, actualPath).first
-                : hashFile(outputHash.hashType, actualPath);
-
-            auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
-
-            // true if either floating CA, or incorrect fixed hash.
-            bool needsMove = true;
-
-            if (auto p = std::get_if<DerivationOutputCAFixed>(& i.second.output)) {
-              Hash & h = p->hash.hash;
-              if (h != h2) {
-
-                /* Throw an error after registering the path as
-                   valid. */
-                worker.hashMismatch = true;
-                delayedException = std::make_exception_ptr(
-                    BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        worker.store.printStorePath(dest),
-                        h.to_string(SRI, true),
-                        h2.to_string(SRI, true)));
-              } else {
-                  // matched the fixed hash, so no move needed.
-                  needsMove = false;
-              }
+            rewriteOutput();
+            /* FIXME optimize and deduplicate with addToStore */
+            std::string oldHashPart { scratchPath.hashPart() };
+            HashModuloSink caSink { outputHash.hashType, oldHashPart };
+            switch (outputHash.method) {
+            case FileIngestionMethod::Recursive:
+                dumpPath(actualPath, caSink);
+                break;
+            case FileIngestionMethod::Flat:
+                readFile(actualPath, caSink);
+                break;
             }
-
-            if (needsMove) {
-                Path actualDest = worker.store.Store::toRealPath(dest);
-
-                if (worker.store.isValidPath(dest))
-                    std::rethrow_exception(delayedException);
-
-                if (actualPath != actualDest) {
-                    PathLocks outputLocks({actualDest});
-                    deletePath(actualDest);
-                    if (rename(actualPath.c_str(), actualDest.c_str()) == -1)
-                        throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest));
-                }
-
-                path = worker.store.printStorePath(dest);
-                actualPath = actualDest;
-            }
-            else
-                assert(worker.store.parseStorePath(path) == dest);
-
-            ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = h2,
+            auto got = caSink.finish().first;
+            auto refs = rewriteRefs();
+            ValidPathInfo newInfo0 {
+                worker.store.makeFixedOutputPath(
+                    outputHash.method,
+                    got,
+                    outputPathName(drv->name, outputName),
+                    refs.second,
+                    refs.first),
             };
+            newInfo0.ca = FixedOutputHash {
+                .method = outputHash.method,
+                .hash = got,
+            };
+            newInfo0.references = refs.second;
+            if (refs.first)
+                newInfo0.references.insert(newInfo0.path);
+            {
+                HashModuloSink narSink { htSHA256, oldHashPart };
+                dumpPath(actualPath, narSink);
+                auto narHashAndSize = narSink.finish();
+                newInfo0.narHash = narHashAndSize.first;
+                newInfo0.narSize = narHashAndSize.second;
+            }
+
+            /* Check wanted hash if output is fixed */
+            if (auto p = std::get_if<DerivationOutputCAFixed>(&output.output)) {
+                Hash & wanted = p->hash.hash;
+                if (wanted != got) {
+                    /* Throw an error after registering the path as
+                       valid. */
+                    worker.hashMismatch = true;
+                    delayedException = std::make_exception_ptr(
+                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
+                            worker.store.printStorePath(drvPath),
+                            wanted.to_string(SRI, true),
+                            got.to_string(SRI, true)));
+                }
+            }
+
+            assert(newInfo0.ca);
+            return newInfo0;
+        }}();
+
+        /* Calculate where we'll move the output files. In the checking case we
+           will leave leave them where they are, for now, rather than move to
+           their usual "final destination" */
+        auto finalDestPath = worker.store.printStorePath(newInfo.path);
+
+        /* Lock final output path, if not already locked. This happens with
+           floating CA derivations and hash-mismatching fixed-output
+           derivations. */
+        PathLocks dynamicOutputLock;
+        auto optFixedPath = output.pathOpt(worker.store, drv->name);
+        if (!optFixedPath ||
+            worker.store.printStorePath(*optFixedPath) != finalDestPath)
+        {
+            assert(newInfo.ca);
+            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
+        }
+
+        /* Move files, if needed */
+        if (worker.store.toRealPath(finalDestPath) != actualPath) {
+            if (buildMode == bmRepair) {
+                /* Path already exists, need to replace it */
+                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
+                actualPath = worker.store.toRealPath(finalDestPath);
+            } else if (buildMode == bmCheck) {
+                /* Path already exists, and we want to compare, so we leave out
+                   new path in place. */
+            } else if (finalDestPath == finalDestPath && worker.store.isValidPath(newInfo.path)) {
+                /* Path already exists because CA path produced by something
+                   else. No moving needed. */
+                assert(newInfo.ca);
+            } else {
+                /* Temporarily add write perm so we can move, will be fixed
+                   later. */
+                {
+                    struct stat st;
+                    auto & mode = st.st_mode;
+                    if (lstat(actualPath.c_str(), &st))
+                        throw SysError("getting attributes of path '%1%'", actualPath);
+                    mode |= 0200;
+                    if (chmod(actualPath.c_str(), mode) == -1)
+                        throw SysError("changing mode of '%1%' to %2$o", actualPath, mode);
+                }
+                if (rename(
+                        actualPath.c_str(),
+                        worker.store.toRealPath(finalDestPath).c_str()) == -1)
+                    throw SysError("moving build output '%1%' from it's temporary location to the Nix store", finalDestPath);
+                actualPath = worker.store.toRealPath(finalDestPath);
+            }
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -3810,45 +4116,33 @@ void DerivationGoal::registerOutputs()
         canonicalisePathMetaData(actualPath,
             buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen);
 
-        /* For this output path, find the references to other paths
-           contained in it.  Compute the SHA-256 NAR hash at the same
-           time.  The hash is stored in the database so that we can
-           verify later on whether nobody has messed with the store. */
-        debug("scanning for references inside '%1%'", path);
-        // HashResult hash;
-        auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths));
-        auto references = worker.store.parseStorePathSet(pathSetAndHash.first);
-        HashResult hash = pathSetAndHash.second;
-
         if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
-            ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path)));
-            if (hash.first != info.narHash) {
+            if (!worker.store.isValidPath(newInfo.path)) continue;
+            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
+            if (newInfo.narHash != oldInfo.narHash) {
                 worker.checkMismatch = true;
                 if (settings.runDiffHook || settings.keepFailed) {
-                    Path dst = worker.store.toRealPath(path + checkSuffix);
+                    Path dst = worker.store.toRealPath(finalDestPath + checkSuffix);
                     deletePath(dst);
                     moveCheckToStore(actualPath, dst);
 
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),
                         buildUser ? buildUser->getGID() : getgid(),
-                        path, dst, worker.store.printStorePath(drvPath), tmpDir);
+                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
 
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
                 } else
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
             }
 
             /* Since we verified the build, it's now ultimately trusted. */
-            if (!info.ultimate) {
-                info.ultimate = true;
-                worker.store.signPathInfo(info);
-                ValidPathInfos infos;
-                infos.push_back(std::move(info));
-                worker.store.registerValidPaths(infos);
+            if (!oldInfo.ultimate) {
+                oldInfo.ultimate = true;
+                worker.store.signPathInfo(oldInfo);
+                worker.store.registerValidPaths({ std::move(oldInfo) });
             }
 
             continue;
@@ -3865,24 +4159,22 @@ void DerivationGoal::registerOutputs()
 
         if (curRound == nrRounds) {
             worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(worker.store.parseStorePath(path));
+            worker.markContentsGood(newInfo.path);
         }
 
-        ValidPathInfo info(worker.store.parseStorePath(path));
-        info.narHash = hash.first;
-        info.narSize = hash.second;
-        info.references = std::move(references);
-        info.deriver = drvPath;
-        info.ultimate = true;
-        info.ca = ca;
-        worker.store.signPathInfo(info);
+        newInfo.deriver = drvPath;
+        newInfo.ultimate = true;
+        worker.store.signPathInfo(newInfo);
 
-        if (!info.references.empty()) {
-            // FIXME don't we have an experimental feature for fixed output with references?
-            info.ca = {};
-        }
+        finish(newInfo.path);
 
-        infos.emplace(i.first, std::move(info));
+        /* If it's a CA path, register it right away. This is necessary if it
+           isn't statically known so that we can safely unlock the path before
+           the next iteration */
+        if (newInfo.ca)
+            worker.store.registerValidPaths({newInfo});
+
+        infos.emplace(outputName, std::move(newInfo));
     }
 
     if (buildMode == bmCheck) return;
@@ -3925,8 +4217,8 @@ void DerivationGoal::registerOutputs()
 
     /* If this is the first round of several, then move the output out of the way. */
     if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & i : drv->outputs) {
-            auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
+        for (auto & [_, outputStorePath] : finalOutputs) {
+            auto path = worker.store.printStorePath(outputStorePath);
             Path prev = path + checkSuffix;
             deletePath(prev);
             Path dst = path + checkSuffix;
@@ -3943,8 +4235,8 @@ void DerivationGoal::registerOutputs()
     /* Remove the .check directories if we're done. FIXME: keep them
        if the result was not determistic? */
     if (curRound == nrRounds) {
-        for (auto & i : drv->outputs) {
-            Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
+        for (auto & [_, outputStorePath] : finalOutputs) {
+            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
             deletePath(prev);
         }
     }
@@ -3954,7 +4246,13 @@ void DerivationGoal::registerOutputs()
        outputs, this will fail. */
     {
         ValidPathInfos infos2;
-        for (auto & i : infos) infos2.push_back(i.second);
+        for (auto & [outputName, newInfo] : infos) {
+            /* FIXME: we will want to track this mapping in the DB whether or
+               not we have a drv file. */
+            if (useDerivation)
+                worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
+            infos2.push_back(newInfo);
+        }
         worker.store.registerValidPaths(infos2);
     }
 
@@ -4030,7 +4328,17 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
             {
                 if (!value) return;
 
-                auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);
+                /* Parse a list of reference specifiers.  Each element must
+                   either be a store path, or the symbolic name of the output
+                   of the derivation (such as `out'). */
+                StorePathSet spec;
+                for (auto & i : *value) {
+                    if (worker.store.isStorePath(i))
+                        spec.insert(worker.store.parseStorePath(i));
+                    else if (finalOutputs.count(i))
+                        spec.insert(finalOutputs.at(i));
+                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
+                }
 
                 auto used = recursive
                     ? getClosure(info.path).first
@@ -4239,31 +4547,65 @@ void DerivationGoal::flushLine()
 }
 
 
-StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
+std::map<std::string, std::optional<StorePath>> DerivationGoal::queryDerivationOutputMap()
 {
-    StorePathSet result;
-    for (auto & i : drv->outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        bool good =
-            worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
-            (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
-        if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
+    if (drv->type() != DerivationType::CAFloating) {
+        std::map<std::string, std::optional<StorePath>> res;
+        for (auto & [name, output] : drv->outputs)
+            res.insert_or_assign(name, output.pathOpt(worker.store, drv->name));
+        return res;
+    } else {
+        return worker.store.queryDerivationOutputMap(drvPath);
+    }
+}
+
+OutputPathMap DerivationGoal::queryDerivationOutputMapAssumeTotal()
+{
+    if (drv->type() != DerivationType::CAFloating) {
+        OutputPathMap res;
+        for (auto & [name, output] : drv->outputs)
+            res.insert_or_assign(name, *output.pathOpt(worker.store, drv->name));
+        return res;
+    } else {
+        return worker.store.queryDerivationOutputMapAssumeTotal(drvPath);
     }
-    return result;
 }
 
 
-void DerivationGoal::addHashRewrite(const StorePath & path)
+void DerivationGoal::checkPathValidity()
 {
-    auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32));
-    auto p = worker.store.makeStorePath(
+    bool checkHash = buildMode == bmRepair;
+    for (auto & i : queryDerivationOutputMap()) {
+        InitialOutputStatus status {
+            .wanted = wantOutput(i.first, wantedOutputs),
+        };
+        if (i.second) {
+            auto outputPath = *i.second;
+            status.known = {
+                .path = outputPath,
+                .valid = !worker.store.isValidPath(outputPath)
+                    ? std::optional<bool> {}
+                    : !checkHash || worker.pathContentsGood(outputPath),
+            };
+        }
+        initialOutputs.insert_or_assign(i.first, status);
+    }
+}
+
+
+StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
+{
+    return worker.store.makeStorePath(
+        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
+        Hash(htSHA256), outputPathName(drv->name, outputName));
+}
+
+
+StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
+{
+    return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
         Hash(htSHA256), path.name());
-    auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32));
-    deletePath(worker.store.printStorePath(p));
-    inputRewrites[h1] = h2;
-    outputRewrites[h2] = h1;
-    redirectedOutputs.insert_or_assign(path, std::move(p));
 }
 
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 68b081058..58c67e40c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -15,7 +15,8 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
         },
         [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
             return {
-                store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
+                // FIXME if we intend to support multiple CA outputs.
+                dof.path(store, drvName, "out")
             };
         },
         [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
@@ -25,6 +26,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
 }
 
 
+StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+    return store.makeFixedOutputPath(
+        hash.method, hash.hash,
+        outputPathName(drvName, outputName));
+}
+
+
 bool derivationIsCA(DerivationType dt) {
     switch (dt) {
     case DerivationType::InputAddressed: return false;
@@ -106,12 +114,15 @@ static string parseString(std::istream & str)
     return res;
 }
 
+static void validatePath(std::string_view s) {
+    if (s.size() == 0 || s[0] != '/')
+        throw FormatError("bad path '%1%' in derivation", s);
+}
 
 static Path parsePath(std::istream & str)
 {
-    string s = parseString(str);
-    if (s.size() == 0 || s[0] != '/')
-        throw FormatError("bad path '%1%' in derivation", s);
+    auto s = parseString(str);
+    validatePath(s);
     return s;
 }
 
@@ -141,7 +152,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
 {
-    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+    expect(str, ","); auto pathS = parseString(str);
     expect(str, ","); auto hashAlgo = parseString(str);
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
@@ -152,30 +163,35 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
             method = FileIngestionMethod::Recursive;
             hashAlgo = string(hashAlgo, 2);
         }
-        const HashType hashType = parseHashType(hashAlgo);
-
-        return hash != ""
-            ? DerivationOutput {
-                  .output = DerivationOutputCAFixed {
-                      .hash = FixedOutputHash {
-                          .method = std::move(method),
-                          .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
-                      },
-                  }
-               }
-            : (settings.requireExperimentalFeature("ca-derivations"),
-              DerivationOutput {
-                  .output =  DerivationOutputCAFloating {
-                      .method = std::move(method),
-                      .hashType = std::move(hashType),
-                  },
-              });
-    } else
+        const auto hashType = parseHashType(hashAlgo);
+        if (hash != "") {
+            validatePath(pathS);
+            return DerivationOutput {
+                .output = DerivationOutputCAFixed {
+                    .hash = FixedOutputHash {
+                        .method = std::move(method),
+                        .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
+                    },
+                },
+            };
+        } else {
+            settings.requireExperimentalFeature("ca-derivations");
+            assert(pathS == "");
+            return DerivationOutput {
+                .output = DerivationOutputCAFloating {
+                    .method = std::move(method),
+                    .hashType = std::move(hashType),
+                },
+            };
+        }
+    } else {
+        validatePath(pathS);
         return DerivationOutput {
             .output = DerivationOutputInputAddressed {
-                .path = std::move(path),
+                .path = store.parseStorePath(pathS),
             }
         };
+    }
 }
 
 
@@ -316,17 +332,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
-        s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
         std::visit(overloaded {
             [&](DerivationOutputInputAddressed doi) {
+                s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
             },
             [&](DerivationOutputCAFixed dof) {
+                s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
                 s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
                 s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
             },
             [&](DerivationOutputCAFloating dof) {
+                s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                 s += ','; printUnquotedString(s, "");
             },
@@ -382,6 +400,16 @@ bool isDerivation(const string & fileName)
 }
 
 
+std::string outputPathName(std::string_view drvName, std::string_view outputName) {
+    std::string res { drvName };
+    if (outputName != "out") {
+        res += "-";
+        res += outputName;
+    }
+    return res;
+}
+
+
 DerivationType BasicDerivation::type() const
 {
     std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
@@ -479,7 +507,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
             auto hash = hashString(htSHA256, "fixed:out:"
                 + dof.hash.printMethodAlgo() + ":"
                 + dof.hash.hash.to_string(Base16, false) + ":"
-                + store.printStorePath(i.second.path(store, drv.name)));
+                + store.printStorePath(dof.path(store, drv.name, i.first)));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
         return outputHashes;
@@ -530,17 +558,9 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 }
 
 
-StorePathSet BasicDerivation::outputPaths(const Store & store) const
-{
-    StorePathSet paths;
-    for (auto & i : outputs)
-        paths.insert(i.second.path(store, name));
-    return paths;
-}
-
 static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
-    auto path = store.parseStorePath(readString(in));
+    auto pathS = readString(in);
     auto hashAlgo = readString(in);
     auto hash = readString(in);
 
@@ -550,29 +570,35 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
             method = FileIngestionMethod::Recursive;
             hashAlgo = string(hashAlgo, 2);
         }
-        auto hashType = parseHashType(hashAlgo);
-        return hash != ""
-            ? DerivationOutput {
-                  .output = DerivationOutputCAFixed {
-                      .hash = FixedOutputHash {
-                          .method = std::move(method),
-                          .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
-                      },
-                  }
-               }
-            : (settings.requireExperimentalFeature("ca-derivations"),
-              DerivationOutput {
-                  .output = DerivationOutputCAFloating {
-                      .method = std::move(method),
-                      .hashType = std::move(hashType),
-                  },
-              });
-    } else
+        const auto hashType = parseHashType(hashAlgo);
+        if (hash != "") {
+            validatePath(pathS);
+            return DerivationOutput {
+                .output = DerivationOutputCAFixed {
+                    .hash = FixedOutputHash {
+                        .method = std::move(method),
+                        .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
+                    },
+                },
+            };
+        } else {
+            settings.requireExperimentalFeature("ca-derivations");
+            assert(pathS == "");
+            return DerivationOutput {
+                .output = DerivationOutputCAFloating {
+                    .method = std::move(method),
+                    .hashType = std::move(hashType),
+                },
+            };
+        }
+    } else {
+        validatePath(pathS);
         return DerivationOutput {
             .output = DerivationOutputInputAddressed {
-                .path = std::move(path),
+                .path = store.parseStorePath(pathS),
             }
         };
+    }
 }
 
 StringSet BasicDerivation::outputNames() const
@@ -624,18 +650,21 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs) {
-        out << i.first
-            << store.printStorePath(i.second.path(store, drv.name));
+        out << i.first;
         std::visit(overloaded {
             [&](DerivationOutputInputAddressed doi) {
-                out << "" << "";
+                out << store.printStorePath(doi.path)
+                    << ""
+                    << "";
             },
             [&](DerivationOutputCAFixed dof) {
-                out << dof.hash.printMethodAlgo()
+                out << store.printStorePath(dof.path(store, drv.name, i.first))
+                    << dof.hash.printMethodAlgo()
                     << dof.hash.hash.to_string(Base16, false);
             },
             [&](DerivationOutputCAFloating dof) {
-                out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+                out << ""
+                    << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
                     << "";
             },
         }, i.second.output);
@@ -654,5 +683,14 @@ std::string hashPlaceholder(const std::string & outputName)
     return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
 }
 
+StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
+{
+    auto drvNameWithExtension = drvPath.name();
+    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
+    return store.makeStorePath(
+        "downstream-placeholder:" + std::string { drvPath.name() } + ":" + std::string { outputName },
+        "compressed:" + std::string { drvPath.hashPart() },
+        outputPathName(drvName, outputName));
+}
 
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 14e0e947a..963b44b8c 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
 struct DerivationOutputCAFixed
 {
     FixedOutputHash hash; /* hash used for expected hash computation */
+    StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 };
 
 /* Floating-output derivations, whose output paths are content addressed, but
@@ -48,12 +49,6 @@ struct DerivationOutput
     > output;
     std::optional<HashType> hashAlgoOpt(const Store & store) const;
     std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
-    /* DEPRECATED: Remove after CA drvs are fully implemented */
-    StorePath path(const Store & store, std::string_view drvName) const {
-        auto p = pathOpt(store, drvName);
-        if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
-        return *p;
-    }
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -101,9 +96,6 @@ struct BasicDerivation
     /* Return true iff this is a fixed-output derivation. */
     DerivationType type() const;
 
-    /* Return the output paths of a derivation. */
-    StorePathSet outputPaths(const Store & store) const;
-
     /* Return the output names of a derivation. */
     StringSet outputNames() const;
 
@@ -136,6 +128,8 @@ Derivation readDerivation(const Store & store, const Path & drvPath, std::string
 // FIXME: remove
 bool isDerivation(const string & fileName);
 
+std::string outputPathName(std::string_view drvName, std::string_view outputName);
+
 // known CA drv's output hashes, current just for fixed-output derivations
 // whose output hashes are always known since they are fixed up-front.
 typedef std::map<std::string, Hash> CaOutputHashes;
@@ -185,4 +179,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 
 std::string hashPlaceholder(const std::string & outputName);
 
+StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
+
 }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e96091aae..027efc7c9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -578,13 +578,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
                 envHasRightPath(path, i.first);
             },
             [&](DerivationOutputCAFloating _) {
-                throw UnimplementedError("floating CA output derivations are not yet implemented");
+                /* Nothing to check */
             },
         }, i.second.output);
     }
 }
 
 
+void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
+{
+    auto state(_state.lock());
+    return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
+}
+
+void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
+{
+    retrySQLite<void>([&]() {
+        state.stmtAddDerivationOutput.use()
+            (deriver)
+            (outputName)
+            (printStorePath(output))
+            .exec();
+    });
+
+}
+
+
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
@@ -619,11 +638,11 @@ uint64_t LocalStore::addValidPath(State & state,
         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 
         for (auto & i : drv.outputs) {
-            state.stmtAddDerivationOutput.use()
-                (id)
-                (i.first)
-                (printStorePath(i.second.path(*this, drv.name)))
-                .exec();
+            /* Floating CA derivations have indeterminate output paths until
+               they are built, so don't register anything in that case */
+            auto optPath = i.second.pathOpt(*this, drv.name);
+            if (optPath)
+                linkDeriverToPath(state, id, i.first, *optPath);
         }
     }
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 627b1f557..bf4016b13 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -282,6 +282,12 @@ private:
        specified by the ‘secret-key-files’ option. */
     void signPathInfo(ValidPathInfo & info);
 
+    /* Add a mapping from the deriver of the path info (if specified) to its
+     * out path
+     */
+    void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
+    void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
+
     Path getRealStoreDir() override { return realStoreDir; }
 
     void createUser(const std::string & userName, uid_t userId) override;
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 0ae1ceaad..9024fb2bd 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 return;
             }
 
+            PathSet invalid;
+            /* true for regular derivations, and CA derivations for which we
+               have a trust mapping for all wanted outputs. */
+            auto knownOutputPaths = true;
+            for (auto & [outputName, pathOpt] : queryDerivationOutputMap(path.path)) {
+                if (!pathOpt) {
+                    knownOutputPaths = false;
+                    break;
+                }
+                if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
+                    invalid.insert(printStorePath(*pathOpt));
+            }
+            if (knownOutputPaths && invalid.empty()) return;
+
             auto drv = make_ref<Derivation>(derivationFromPath(path.path));
             ParsedDerivation parsedDrv(StorePath(path.path), *drv);
 
-            PathSet invalid;
-            for (auto & j : drv->outputs)
-                if (wantOutput(j.first, path.outputs)
-                    && !isValidPath(j.second.path(*this, drv->name)))
-                    invalid.insert(printStorePath(j.second.path(*this, drv->name)));
-            if (invalid.empty()) return;
-
-            if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
+            if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
                     pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 62a3cda61..d2096cb49 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
 std::pair<PathSet, HashResult> scanForReferences(const string & path,
     const PathSet & refs)
 {
-    RefScanSink refsSink;
     HashSink hashSink { htSHA256 };
-    TeeSink sink { refsSink, hashSink };
+    auto found = scanForReferences(hashSink, path, refs);
+    auto hash = hashSink.finish();
+    return std::pair<PathSet, HashResult>(found, hash);
+}
+
+PathSet scanForReferences(Sink & toTee,
+    const string & path, const PathSet & refs)
+{
+    RefScanSink refsSink;
+    TeeSink sink { refsSink, toTee };
     std::map<string, Path> backMap;
 
     /* For efficiency (and a higher hit rate), just search for the
@@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
         found.insert(j->second);
     }
 
-    auto hash = hashSink.finish();
-
-    return std::pair<PathSet, HashResult>(found, hash);
+    return found;
 }
 
 
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index 598a3203a..c2efd095c 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -7,6 +7,8 @@ namespace nix {
 
 std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
 
+PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
+
 struct RewritingSink : Sink
 {
     std::string from, to, prev;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 005415666..e69d15c53 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -74,7 +74,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
 template<>
 std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
 {
-	auto s = readString(from);
+    auto s = readString(from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e66c04df4..63accd514 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -140,21 +140,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
 */
 
 
-StorePath Store::makeStorePath(const string & type,
-    const Hash & hash, std::string_view name) const
+StorePath Store::makeStorePath(std::string_view type,
+    std::string_view hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
+    string s = std::string { type } + ":" + std::string { hash }
+        + ":" + storeDir + ":" + std::string { name };
     auto h = compressHash(hashString(htSHA256, s), 20);
     return StorePath(h, name);
 }
 
 
-StorePath Store::makeOutputPath(const string & id,
+StorePath Store::makeStorePath(std::string_view type,
     const Hash & hash, std::string_view name) const
 {
-    return makeStorePath("output:" + id, hash,
-        std::string(name) + (id == "out" ? "" : "-" + id));
+    return makeStorePath(type, hash.to_string(Base16, true), name);
+}
+
+
+StorePath Store::makeOutputPath(std::string_view id,
+    const Hash & hash, std::string_view name) const
+{
+    return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9003ab541..943f08211 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -244,10 +244,12 @@ public:
     StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
 
     /* Constructs a unique store path name. */
-    StorePath makeStorePath(const string & type,
+    StorePath makeStorePath(std::string_view type,
+        std::string_view hash, std::string_view name) const;
+    StorePath makeStorePath(std::string_view type,
         const Hash & hash, std::string_view name) const;
 
-    StorePath makeOutputPath(const string & id,
+    StorePath makeOutputPath(std::string_view id,
         const Hash & hash, std::string_view name) const;
 
     StorePath makeFixedOutputPath(FileIngestionMethod method,
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index c29c6b29b..3650857aa 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -22,6 +22,12 @@ struct Sink
     }
 };
 
+/* Just throws away data. */
+struct NullSink : Sink
+{
+    void operator () (const unsigned char * data, size_t len) override
+    { }
+};
 
 /* A buffered abstract sink. */
 struct BufferedSink : virtual Sink
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 94412042f..be73ea724 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -487,50 +487,56 @@ static void _main(int argc, char * * argv)
 
         std::vector<StorePathWithOutputs> pathsToBuild;
 
-        std::map<Path, Path> drvPrefixes;
-        std::map<Path, Path> resultSymlinks;
-        std::vector<Path> outPaths;
+        std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
 
         for (auto & drvInfo : drvs) {
-            auto drvPath = drvInfo.queryDrvPath();
-            auto outPath = drvInfo.queryOutPath();
+            auto drvPath = store->parseStorePath(drvInfo.queryDrvPath());
 
             auto outputName = drvInfo.queryOutputName();
             if (outputName == "")
-                throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath);
+                throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
-            pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}});
+            pathsToBuild.push_back({drvPath, {outputName}});
 
-            std::string drvPrefix;
-            auto i = drvPrefixes.find(drvPath);
-            if (i != drvPrefixes.end())
-                drvPrefix = i->second;
+            auto i = drvMap.find(drvPath);
+            if (i != drvMap.end())
+                i->second.second.insert(outputName);
             else {
-                drvPrefix = outLink;
-                if (drvPrefixes.size())
-                    drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
-                drvPrefixes[drvPath] = drvPrefix;
+                drvMap[drvPath] = {drvMap.size(), {outputName}};
             }
-
-            std::string symlink = drvPrefix;
-            if (outputName != "out") symlink += "-" + outputName;
-
-            resultSymlinks[symlink] = outPath;
-            outPaths.push_back(outPath);
         }
 
         buildPaths(pathsToBuild);
 
         if (dryRun) return;
 
-        for (auto & symlink : resultSymlinks)
-            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true);
+        std::vector<StorePath> outPaths;
+
+        for (auto & [drvPath, info] : drvMap) {
+            auto & [counter, wantedOutputs] = info;
+            std::string drvPrefix = outLink;
+            if (counter)
+                drvPrefix += fmt("-%d", counter + 1);
+
+            auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(drvPath);
+
+            for (auto & outputName : wantedOutputs) {
+                auto outputPath = builtOutputs.at(outputName);
+
+                if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+                    std::string symlink = drvPrefix;
+                    if (outputName != "out") symlink += "-" + outputName;
+                    store2->addPermRoot(outputPath, absPath(symlink), true);
+                }
+
+                outPaths.push_back(outputPath);
+            }
+        }
 
         logger->stop();
 
         for (auto & path : outPaths)
-            std::cout << path << '\n';
+            std::cout << store->printStorePath(path) << '\n';
     }
 }
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 2996e36c4..9eadf62e4 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -65,6 +65,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
 
     if (path.path.isDerivation()) {
         if (build) store->buildPaths({path});
+        auto outputPaths = store->queryDerivationOutputMapAssumeTotal(path.path);
         Derivation drv = store->derivationFromPath(path.path);
         rootNr++;
 
@@ -77,7 +78,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
             if (i == drv.outputs.end())
                 throw Error("derivation '%s' does not have an output named '%s'",
                     store2->printStorePath(path.path), j);
-            auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
+            auto outPath = outputPaths.at(i->first);
+            auto retPath = store->printStorePath(outPath);
             if (store2) {
                 if (gcRoot == "")
                     printGCWarning();
@@ -85,10 +87,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                     Path rootName = gcRoot;
                     if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     if (i->first != "out") rootName += "-" + i->first;
-                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot);
+                    retPath = store2->addPermRoot(outPath, rootName, indirectRoot);
                 }
             }
-            outputs.insert(outPath);
+            outputs.insert(retPath);
         }
         return outputs;
     }
@@ -218,8 +220,14 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
     if (useOutput && storePath.isDerivation()) {
         auto drv = store->derivationFromPath(storePath);
         StorePathSet outputs;
-        for (auto & i : drv.outputs)
-            outputs.insert(i.second.path(*store, drv.name));
+        if (forceRealise)
+            return store->queryDerivationOutputs(storePath);
+        for (auto & i : drv.outputs) {
+            auto optPath = i.second.pathOpt(*store, drv.name);
+            if (!optPath)
+                throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
+            outputs.insert(*optPath);
+        }
         return outputs;
     }
     else return {storePath};
@@ -309,11 +317,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
 
         case qOutputs: {
             for (auto & i : opArgs) {
-                auto i2 = store->followLinksToStorePath(i);
-                if (forceRealise) realisePath({i2});
-                Derivation drv = store->derivationFromPath(i2);
-                for (auto & j : drv.outputs)
-                    cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
+                auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise);
+                for (auto & outputPath : outputs)
+                    cout << fmt("%1%\n", store->printStorePath(outputPath));
             }
             break;
         }
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 13d14a7fb..a66e8dac4 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
                             store2->addPermRoot(bo.path, absPath(symlink), true);
                         },
                         [&](BuildableFromDrv bfd) {
-                            for (auto & output : bfd.outputs) {
+                            auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(bfd.drvPath);
+                            for (auto & output : builtOutputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);
diff --git a/src/nix/command.cc b/src/nix/command.cc
index da32819da..4a93d8e73 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -137,7 +137,9 @@ void MixProfile::updateProfile(const Buildables & buildables)
             },
             [&](BuildableFromDrv bfd) {
                 for (auto & output : bfd.outputs) {
-                    result.push_back(output.second);
+                    if (!output.second)
+                        throw Error("output path should be known because we just tried to build it");
+                    result.push_back(*output.second);
                 }
             },
         }, buildable);
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index ea152709f..2ff94cd38 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -302,10 +302,10 @@ struct InstallableStorePath : Installable
     Buildables toBuildables() override
     {
         if (storePath.isDerivation()) {
-            std::map<std::string, StorePath> outputs;
+            std::map<std::string, std::optional<StorePath>> outputs;
             auto drv = store->readDerivation(storePath);
             for (auto & [name, output] : drv.outputs)
-                outputs.emplace(name, output.path(*store, drv.name));
+                outputs.emplace(name, output.pathOpt(*store, drv.name));
             return {
                 BuildableFromDrv {
                     .drvPath = storePath,
@@ -331,7 +331,7 @@ Buildables InstallableValue::toBuildables()
 {
     Buildables res;
 
-    std::map<StorePath, OutputPathMap> drvsToOutputs;
+    std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
@@ -674,8 +674,11 @@ StorePathSet toStorePaths(ref<Store> store,
                     outPaths.insert(bo.path);
                 },
                 [&](BuildableFromDrv bfd) {
-                    for (auto & output : bfd.outputs)
-                        outPaths.insert(output.second);
+                    for (auto & output : bfd.outputs) {
+                        if (!output.second)
+                            throw Error("Cannot operate on output of unbuilt CA drv");
+                        outPaths.insert(*output.second);
+                    }
                 },
             }, b);
     } else {
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index 26e87ee3a..41c75a4ed 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -20,7 +20,7 @@ struct BuildableOpaque {
 
 struct BuildableFromDrv {
     StorePath drvPath;
-    std::map<std::string, StorePath> outputs;
+    std::map<std::string, std::optional<StorePath>> outputs;
 };
 
 typedef std::variant<
@@ -82,7 +82,7 @@ struct InstallableValue : Installable
     struct DerivationInfo
     {
         StorePath drvPath;
-        StorePath outPath;
+        std::optional<StorePath> outPath;
         std::string outputName;
     };
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 7dcc0b6d4..be002519a 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -178,7 +178,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                 auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
 
                 ProfileElement element;
-                element.storePaths = {drv.outPath}; // FIXME
+                if (!drv.outPath)
+                    throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
+                element.storePaths = {*drv.outPath}; // FIXME
                 element.source = ProfileElementSource{
                     installable2->flakeRef,
                     resolvedRef,
@@ -189,7 +191,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 
                 manifest.elements.emplace_back(std::move(element));
             } else
-                throw Error("'nix profile install' does not support argument '%s'", installable->what());
+                throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
         }
 
         store->buildPaths(pathsToBuild);
@@ -347,7 +349,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 printInfo("upgrading '%s' from flake '%s' to '%s'",
                     element.source->attrPath, element.source->resolvedRef, resolvedRef);
 
-                element.storePaths = {drv.outPath}; // FIXME
+                if (!drv.outPath)
+                    throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
+                element.storePaths = {*drv.outPath}; // FIXME
                 element.source = ProfileElementSource{
                     installable.flakeRef,
                     resolvedRef,
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index c3c9e54a8..0224f891d 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -488,10 +488,10 @@ bool NixRepl::processLine(string line)
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
             if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
+                auto outputs = state->store->queryDerivationOutputMapAssumeTotal(state->store->parseStorePath(drvPath));
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-                for (auto & i : drv.outputs)
-                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
+                for (auto & i : outputs)
+                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second));
             }
         } else if (command == ":i") {
             runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 1b51d114f..b204b2d4c 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -69,12 +69,12 @@ struct CmdShowDerivation : InstallablesCommand
                 auto outputsObj(drvObj.object("outputs"));
                 for (auto & output : drv.outputs) {
                     auto outputObj(outputsObj.object(output.first));
-                    outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
-
                     std::visit(overloaded {
                         [&](DerivationOutputInputAddressed doi) {
+                            outputObj.attr("path", store->printStorePath(doi.path));
                         },
                         [&](DerivationOutputCAFixed dof) {
+                            outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, output.first)));
                             outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
                             outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
                         },
diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
new file mode 100644
index 000000000..586e4cba6
--- /dev/null
+++ b/tests/content-addressed.nix
@@ -0,0 +1,19 @@
+with import ./config.nix;
+
+{ seed ? 0 }:
+# A simple content-addressed derivation.
+# The derivation can be arbitrarily modified by passing a different `seed`,
+# but the output will always be the same
+mkDerivation {
+  name = "simple-content-addressed";
+  buildCommand = ''
+    set -x
+    echo "Building a CA derivation"
+    echo "The seed is ${toString seed}"
+    mkdir -p $out
+    echo "Hello World" > $out/hello
+  '';
+  __contentAddressed = true;
+  outputHashMode = "recursive";
+  outputHashAlgo = "sha256";
+}
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
new file mode 100644
index 000000000..2968f3a8c
--- /dev/null
+++ b/tests/content-addressed.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+clearStore
+clearCache
+
+export REMOTE_STORE=file://$cacheDir
+
+drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1)
+nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
+
+out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link)
+out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link)
+
+test $out1 == $out2
diff --git a/tests/local.mk b/tests/local.mk
index 0f3bfe606..4f4e63a5a 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -32,7 +32,8 @@ nix_tests = \
   post-hook.sh \
   function-trace.sh \
   recursive.sh \
-  flakes.sh
+  flakes.sh \
+  content-addressed.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/multiple-outputs.nix b/tests/multiple-outputs.nix
index 4a9010d18..b915493f7 100644
--- a/tests/multiple-outputs.nix
+++ b/tests/multiple-outputs.nix
@@ -2,6 +2,21 @@ with import ./config.nix;
 
 rec {
 
+  # Want to ensure that "out" doesn't get a suffix on it's path.
+  nameCheck = mkDerivation {
+    name = "multiple-outputs-a";
+    outputs = [ "out" "dev" ];
+    builder = builtins.toFile "builder.sh"
+      ''
+        mkdir $first $second
+        test -z $all
+        echo "first" > $first/file
+        echo "second" > $second/file
+        ln -s $first $second/link
+      '';
+    helloString = "Hello, world!";
+  };
+
   a = mkDerivation {
     name = "multiple-outputs-a";
     outputs = [ "first" "second" ];
diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh
index bedbc39a4..7a6ec181d 100644
--- a/tests/multiple-outputs.sh
+++ b/tests/multiple-outputs.sh
@@ -4,6 +4,12 @@ clearStore
 
 rm -f $TEST_ROOT/result*
 
+# Test whether the output names match our expectations
+outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath)
+[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ]
+outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath)
+[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ]
+
 # Test whether read-only evaluation works when referring to the
 # ‘drvPath’ attribute.
 echo "evaluating c..."

From f7696c66e85008e2b60a579bcdf5ff13c141d0af Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 8 Aug 2020 15:48:51 +0000
Subject: [PATCH 083/882] Fix perl FFI for floating ca derivations

Path is null when not known statically.
---
 perl/lib/Nix/Store.xs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index de60fb939..6fe01fff0 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -303,11 +303,15 @@ SV * derivationFromPath(char * drvPath)
             hash = newHV();
 
             HV * outputs = newHV();
-            for (auto & i : drv.outputs)
+            for (auto & i : drv.outputs) {
+                auto pathOpt = i.second.pathOpt(*store(), drv.name);
                 hv_store(
                     outputs, i.first.c_str(), i.first.size(),
-                    newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
+                    !pathOpt
+                        ? newSV(0) /* null value */
+                        : newSVpv(store()->printStorePath(*pathOpt).c_str(), 0),
                     0);
+            }
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
             AV * inputDrvs = newAV();

From bcd0629c2e499788e4b3a9d6e380143a8f807419 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 9 Aug 2020 20:32:35 +0000
Subject: [PATCH 084/882] Remove name parameter from `writeDerivation`

The name is now stored with the derivation itself.
---
 src/libexpr/primops.cc      | 2 +-
 src/libstore/derivations.cc | 4 ++--
 src/libstore/derivations.hh | 2 +-
 src/nix/develop.cc          | 9 +++------
 4 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 65d36ca0e..3955287b7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -839,7 +839,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPath = writeDerivation(state.store, drv, state.repair);
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 68b081058..f7e764a80 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -62,7 +62,7 @@ bool BasicDerivation::isBuiltin() const
 
 
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, std::string_view name, RepairFlag repair)
+    const Derivation & drv, RepairFlag repair)
 {
     auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs)
@@ -70,7 +70,7 @@ StorePath writeDerivation(ref<Store> store,
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    auto suffix = std::string(name) + drvExtension;
+    auto suffix = std::string(drv.name) + drvExtension;
     auto contents = drv.unparse(*store, false);
     return settings.readOnlyMode
         ? store->computeStorePathForText(suffix, contents, references)
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 14e0e947a..53dfc7f13 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -128,7 +128,7 @@ enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
+    const Derivation & drv, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
 Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 12658078a..434088da7 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -124,10 +124,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
-    auto drvName = std::string(drvPath.name());
-    assert(hasSuffix(drvName, ".drv"));
-    drvName.resize(drvName.size() - 4);
-    drvName += "-env";
+    drv.name += "-env";
     for (auto & output : drv.outputs)
         drv.env.erase(output.first);
     drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
@@ -136,12 +133,12 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.env["outputs"] = "out";
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
-    auto shellOutPath = store->makeOutputPath("out", h, drvName);
+    auto shellOutPath = store->makeOutputPath("out", h, drv.name);
     drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
                 .path = shellOutPath
             } });
     drv.env["out"] = store->printStorePath(shellOutPath);
-    auto shellDrvPath2 = writeDerivation(store, drv, drvName);
+    auto shellDrvPath2 = writeDerivation(store, drv);
 
     /* Build the derivation. */
     store->buildPaths({{shellDrvPath2}});

From 581183d4d57d5382e3a375f760f3323ef451f555 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 9 Aug 2020 20:51:34 +0000
Subject: [PATCH 085/882] Deduplicate parsing and reading derivations

---
 src/libstore/derivations.cc | 54 +++++++++++--------------------------
 1 file changed, 16 insertions(+), 38 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index f7e764a80..e9e4b5173 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -139,18 +139,14 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+static DerivationOutput parseDerivationOutput(const Store & store,
+    StorePath path, std::string_view hashAlgo, std::string_view hash)
 {
-    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
-    expect(str, ","); auto hashAlgo = parseString(str);
-    expect(str, ","); const auto hash = parseString(str);
-    expect(str, ")");
-
     if (hashAlgo != "") {
         auto method = FileIngestionMethod::Flat;
         if (string(hashAlgo, 0, 2) == "r:") {
             method = FileIngestionMethod::Recursive;
-            hashAlgo = string(hashAlgo, 2);
+            hashAlgo = hashAlgo.substr(2);
         }
         const HashType hashType = parseHashType(hashAlgo);
 
@@ -178,6 +174,16 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
         };
 }
 
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+{
+    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+    expect(str, ","); const auto hashAlgo = parseString(str);
+    expect(str, ","); const auto hash = parseString(str);
+    expect(str, ")");
+
+    return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
+}
+
 
 static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
 {
@@ -541,38 +547,10 @@ StorePathSet BasicDerivation::outputPaths(const Store & store) const
 static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
     auto path = store.parseStorePath(readString(in));
-    auto hashAlgo = readString(in);
-    auto hash = readString(in);
+    const auto hashAlgo = readString(in);
+    const auto hash = readString(in);
 
-    if (hashAlgo != "") {
-        auto method = FileIngestionMethod::Flat;
-        if (string(hashAlgo, 0, 2) == "r:") {
-            method = FileIngestionMethod::Recursive;
-            hashAlgo = string(hashAlgo, 2);
-        }
-        auto hashType = parseHashType(hashAlgo);
-        return hash != ""
-            ? DerivationOutput {
-                  .output = DerivationOutputCAFixed {
-                      .hash = FixedOutputHash {
-                          .method = std::move(method),
-                          .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
-                      },
-                  }
-               }
-            : (settings.requireExperimentalFeature("ca-derivations"),
-              DerivationOutput {
-                  .output = DerivationOutputCAFloating {
-                      .method = std::move(method),
-                      .hashType = std::move(hashType),
-                  },
-              });
-    } else
-        return DerivationOutput {
-            .output = DerivationOutputInputAddressed {
-                .path = std::move(path),
-            }
-        };
+    return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
 }
 
 StringSet BasicDerivation::outputNames() const

From 5f8ae16c8b08485e7625283ccafb99727cd47693 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 10 Aug 2020 17:44:17 +0200
Subject: [PATCH 086/882] Always reset ANSI colors in progress-bar line

When having a message like `waiting for a machine to build X` and
building with `nix build -L`, the log-prefix is always colored yellow[1]
on a small terminal-width as everything (including the ANSI color-reset) is
stripped away.

To work around that problem, this patch explicitly adds an `ANSI_NORMAL`
to the end of the line.

[1] https://imgur.com/a/FjtJOk3
---
 src/libmain/progress-bar.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 3f7d99a1d..be3c06a38 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -362,7 +362,7 @@ public:
         auto width = getWindowSize().second;
         if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
 
-        writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
+        writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
     }
 
     std::string getStatus(State & state)

From 2a0902634eddded44a1775b275a257c98ca1dec8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 00:12:54 +0000
Subject: [PATCH 087/882] Fix error in merge breaking floating CA drvs

Forgot to add this hunk!
---
 src/libstore/derivations.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 732401dc5..e9f35a6f1 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -192,7 +192,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
 
 static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
 {
-    expect(str, ","); const auto pathS = parsePath(str);
+    expect(str, ","); const auto pathS = parseString(str);
     expect(str, ","); const auto hashAlgo = parseString(str);
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");

From d3fa8c04c68a9532f85a18271c35457981a840ec Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 01:13:26 +0000
Subject: [PATCH 088/882] Simplify code as output env vars are unconditional

Since the jsonObject unique ptr is reset to flush the string to make
`__json`, all these `!jsonObject` conditions will always be true.
---
 src/libexpr/primops.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 65d36ca0e..af751a496 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -781,7 +781,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         Hash h = newHashAllowEmpty(*outputHash, ht);
 
         auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
-        if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
+        drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
                 .output = DerivationOutputCAFixed {
                     .hash = FixedOutputHash {
@@ -795,7 +795,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     else if (contentAddressed) {
         HashType ht = parseHashType(outputHashAlgo);
         for (auto & i : outputs) {
-            if (!jsonObject) drv.env[i] = hashPlaceholder(i);
+            drv.env[i] = hashPlaceholder(i);
             drv.outputs.insert_or_assign(i, DerivationOutput {
                 .output = DerivationOutputCAFloating {
                     .method = ingestionMethod,
@@ -813,7 +813,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            that changes in the set of output names do get reflected in
            the hash. */
         for (auto & i : outputs) {
-            if (!jsonObject) drv.env[i] = "";
+            drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
                 DerivationOutput {
                     .output = DerivationOutputInputAddressed {
@@ -828,7 +828,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         for (auto & i : outputs) {
             auto outPath = state.store->makeOutputPath(i, h, drvName);
-            if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
+            drv.env[i] = state.store->printStorePath(outPath);
             drv.outputs.insert_or_assign(i,
                 DerivationOutput {
                     .output = DerivationOutputInputAddressed {

From 1a281ec07f33baa0d1b10d2760edcc7019caec83 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 11 Aug 2020 10:29:43 -0600
Subject: [PATCH 089/882] demote remote build message to Info

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 5247cefa6..cec7b369b 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -201,7 +201,7 @@ static int _main(int argc, char * * argv)
                             % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
                         }
 
-                        logError({
+                        logErrorInfo(lvlInfo, {
                               .name = "Remote build",
                               .description = "Failed to find a machine for remote build!",
                               .hint = hint

From d0f6e338ddbd4b7e37c2d944c368ca3094619e9f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 16:49:10 -0400
Subject: [PATCH 090/882] Apply suggestions from code review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Thanks!!

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
---
 src/libstore/build.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 7c2225805..b9f79e5b7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -876,7 +876,7 @@ private:
          ends up being. (Note that fixed outputs derivations that produce the
          "wrong" output still install that data under its true content-address.)
      */
-    std::map<std::string, StorePath> finalOutputs;
+    OutputPathMap finalOutputs;
 
     BuildMode buildMode;
 
@@ -1055,7 +1055,7 @@ private:
     StorePath makeFallbackPath(const StorePath & path);
     /* Make a path to another based on the output name alone, if one doesn't
        want to use a random path for CA builds. */
-    StorePath makeFallbackPath(std::string_view path);
+    StorePath makeFallbackPath(std::string_view outputName);
 
     void repairClosure();
 
@@ -1336,7 +1336,7 @@ void DerivationGoal::outputsSubstitutionTried()
 
 void DerivationGoal::gaveUpOnSubstitution()
 {
-    /* Otherwise, at least one of the output paths could not be
+    /* At least one of the output paths could not be
        produced using a substitute.  So we have to build instead. */
 
     /* Make sure checkPathValidity() from now on checks all
@@ -2371,7 +2371,7 @@ void DerivationGoal::startBuilder()
            (typically the dependencies of /bin/sh).  Throw them
            out. */
         for (auto & i : drv->outputs) {
-            /* If the name isn't known a prior (i.e. floating content-addressed
+            /* If the name isn't known a priori (i.e. floating content-addressed
                derivation), the temporary location we use should be fresh and
                never in the sandbox in the first place. */
             auto optPath = i.second.pathOpt(worker.store, drv->name);
@@ -3887,7 +3887,7 @@ void DerivationGoal::registerOutputs()
     for (auto & outputName : sortedOutputNames) {
         auto output = drv->outputs.at(outputName);
         auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
 
         auto finish = [&](StorePath finalStorePath) {
             /* Store the final path */

From 07e3466eb4e6d259dc328a5aec834d42f8aacb60 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 21:16:14 +0000
Subject: [PATCH 091/882] Float comment to out describe `gaveUpOnSubstitution`
 in general

---
 src/libstore/build.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b9f79e5b7..1b55b9826 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1334,11 +1334,10 @@ void DerivationGoal::outputsSubstitutionTried()
     gaveUpOnSubstitution();
 }
 
+/* At least one of the output paths could not be
+   produced using a substitute.  So we have to build instead. */
 void DerivationGoal::gaveUpOnSubstitution()
 {
-    /* At least one of the output paths could not be
-       produced using a substitute.  So we have to build instead. */
-
     /* Make sure checkPathValidity() from now on checks all
        outputs. */
     wantedOutputs.clear();

From 8a068bd0252573750be6ea754f0e3ff502fb6019 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 21:25:40 +0000
Subject: [PATCH 092/882] Remove redundant equality check

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1b55b9826..cc596b063 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4086,7 +4086,7 @@ void DerivationGoal::registerOutputs()
             } else if (buildMode == bmCheck) {
                 /* Path already exists, and we want to compare, so we leave out
                    new path in place. */
-            } else if (finalDestPath == finalDestPath && worker.store.isValidPath(newInfo.path)) {
+            } else if (worker.store.isValidPath(newInfo.path)) {
                 /* Path already exists because CA path produced by something
                    else. No moving needed. */
                 assert(newInfo.ca);

From 6d571390500431904d1ca1621fb6791fc25522f1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 22:34:09 +0000
Subject: [PATCH 093/882] Clarify `outputReferences` variable with
 self-describing type

Thanks for the idea, @Regnat!
---
 src/libstore/build.cc | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cc596b063..142149ae3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3800,7 +3800,9 @@ void DerivationGoal::registerOutputs()
        that are most definitely already installed, we just store their final
        name so we can also use it in rewrites. */
     StringSet outputsToSort;
-    std::map<std::string, std::variant<StorePath, StorePathSet>> outputReferences;
+    struct AlreadyRegistered { StorePath path; };
+    struct PerhapsNeedToRegister { StorePathSet refs; };
+    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
     std::map<std::string, struct stat> outputStats;
     for (auto & [outputName, _] : drv->outputs) {
         auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
@@ -3814,7 +3816,9 @@ void DerivationGoal::registerOutputs()
         initialInfo.wanted = buildMode == bmCheck
             || !(initialInfo.known && initialInfo.known->isValid());
         if (!initialInfo.wanted) {
-            outputReferences.insert_or_assign(outputName, initialInfo.known->path);
+            outputReferencesIfUnregistered.insert_or_assign(
+                outputName,
+                AlreadyRegistered { .path = initialInfo.known->path });
             continue;
         }
 
@@ -3851,28 +3855,29 @@ void DerivationGoal::registerOutputs()
         auto references = worker.store.parseStorePathSet(
             scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
 
-        outputReferences.insert_or_assign(outputName, references);
+        outputReferencesIfUnregistered.insert_or_assign(
+            outputName,
+            PerhapsNeedToRegister { .refs = references });
         outputStats.insert_or_assign(outputName, std::move(st));
     }
 
     auto sortedOutputNames = topoSort(outputsToSort,
         {[&](const std::string & name) {
-            auto x = outputReferences.at(name);
             return std::visit(overloaded {
                 /* Since we'll use the already installed versions of these, we
                    can treat them as leaves and ignore any references they
                    have. */
-                [&](StorePath _) { return StringSet {}; },
-                [&](StorePathSet refs) {
+                [&](AlreadyRegistered _) { return StringSet {}; },
+                [&](PerhapsNeedToRegister refs) {
                     StringSet referencedOutputs;
                     /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs)
+                    for (auto & r : refs.refs)
                         for (auto & [o, p] : scratchOutputs)
                             if (r == p)
                                 referencedOutputs.insert(o);
                     return referencedOutputs;
                 },
-            }, x);
+            }, outputReferencesIfUnregistered.at(name));
         }},
         {[&](const std::string & path, const std::string & parent) {
             // TODO with more -vvvv also show the temporary paths for manual inspection.
@@ -3900,14 +3905,14 @@ void DerivationGoal::registerOutputs()
 
         bool rewritten = false;
         std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](StorePath skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath);
+            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
+                finish(skippedFinalPath.path);
                 return std::nullopt;
             },
-            [&](StorePathSet references) -> std::optional<StorePathSet> {
-                return references;
+            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
+                return r.refs;
             },
-        }, outputReferences.at(outputName));
+        }, outputReferencesIfUnregistered.at(outputName));
 
         if (!referencesOpt)
             continue;

From 4c6aac8fdf2cbc85280115da53acebb910d2e60c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 22:46:05 +0000
Subject: [PATCH 094/882] Clarify comment on sandbox and temp fresh paths

---
 src/libstore/build.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 142149ae3..d2bfc73c9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2370,9 +2370,11 @@ void DerivationGoal::startBuilder()
            (typically the dependencies of /bin/sh).  Throw them
            out. */
         for (auto & i : drv->outputs) {
-            /* If the name isn't known a priori (i.e. floating content-addressed
-               derivation), the temporary location we use should be fresh and
-               never in the sandbox in the first place. */
+            /* If the name isn't known a priori (i.e. floating
+               content-addressed derivation), the temporary location we use
+               should be fresh.  Freshness means it is impossible that the path
+               is already in the sandbox, so we don't need to worry about
+               removing it.  */
             auto optPath = i.second.pathOpt(worker.store, drv->name);
             if (optPath)
                 dirsInChroot.erase(worker.store.printStorePath(*optPath));

From 2de201254e8669a6768bb739ff27fd94a87a71b9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 23:07:50 +0000
Subject: [PATCH 095/882] Don't assume a total output map in two places in
 build.cc

Thanks @regnat for catching one of them. The other follows for many of
the same reasons. I'm find fixing others on a need-to-fix basis,
provided their are no regressions.
---
 src/libstore/build.cc | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d2bfc73c9..c8107c7e8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1389,9 +1389,10 @@ void DerivationGoal::repairClosure()
     std::map<StorePath, StorePath> outputsToDrv;
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryDerivationOutputMapAssumeTotal(i);
+            auto depOutputs = worker.store.queryDerivationOutputMap(i);
             for (auto & j : depOutputs)
-                outputsToDrv.insert_or_assign(j.second, i);
+                if (j.second)
+                    outputsToDrv.insert_or_assign(*j.second, i);
         }
 
     /* Check each path (slow!). */
@@ -1459,11 +1460,16 @@ void DerivationGoal::inputsRealised()
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
             assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryDerivationOutputMapAssumeTotal(depDrvPath);
+            auto outputs = worker.store.queryDerivationOutputMap(depDrvPath);
             for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0)
-                    worker.store.computeFSClosure(outputs.at(j), inputPaths);
-                else
+                if (outputs.count(j) > 0) {
+                    auto optRealizedInput = outputs.at(j);
+                    if (!optRealizedInput)
+                        throw Error(
+                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output.",
+                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
+                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
+                } else
                     throw Error(
                         "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
                         worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));

From 18834f77643f5bd1071e1735f23f925d87cce6e5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 23:44:02 +0000
Subject: [PATCH 096/882] Recheck path validity after acquiring lock

It might have changed, and in any event this is how the cod used to work
so let's just keep it.
---
 src/libstore/build.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index c8107c7e8..49042649e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1547,6 +1547,7 @@ void DerivationGoal::tryToBuild()
        omitted, but that would be less efficient.)  Note that since we
        now hold the locks on the output paths, no other process can
        build this derivation, so no further checks are necessary. */
+    checkPathValidity();
     bool allValid = true;
     for (auto & [_, status] : initialOutputs) {
         if (!status.wanted) continue;

From 5f80aea795d3f6f78d682c7b99453b6fb7c9b6ba Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 12 Aug 2020 02:23:31 +0000
Subject: [PATCH 097/882] Break out lambda so output can be matched just once

This is much better.
---
 src/libstore/build.cc | 86 +++++++++++++++++++++----------------------
 1 file changed, 42 insertions(+), 44 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 49042649e..0ea8f9c97 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3973,42 +3973,7 @@ void DerivationGoal::registerOutputs()
             return res;
         };
 
-        ValidPathInfo newInfo = [&]() {if (auto outputP = std::get_if<DerivationOutputInputAddressed>(&output.output)) {
-            /* input-addressed case */
-            auto requiredFinalPath = outputP->path;
-            /* Preemtively add rewrite rule for final hash, as that is
-               what the NAR hash will use rather than normalized-self references */
-            if (scratchPath != requiredFinalPath)
-                outputRewrites.insert_or_assign(
-                    std::string { scratchPath.hashPart() },
-                    std::string { requiredFinalPath.hashPart() });
-            rewriteOutput();
-            auto narHashAndSize = hashPath(htSHA256, actualPath);
-            ValidPathInfo newInfo0 { requiredFinalPath };
-            newInfo0.narHash = narHashAndSize.first;
-            newInfo0.narSize = narHashAndSize.second;
-            auto refs = rewriteRefs();
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-            return newInfo0;
-        } else {
-            /* content-addressed case */
-            DerivationOutputCAFloating outputHash = std::visit(overloaded {
-                [&](DerivationOutputInputAddressed doi) -> DerivationOutputCAFloating {
-                    // Enclosing `if` handles this case in other branch
-                    throw Error("ought to unreachable, handled in other branch");
-                },
-                [&](DerivationOutputCAFixed dof) {
-                    return DerivationOutputCAFloating {
-                        .method = dof.hash.method,
-                        .hashType = dof.hash.hash.type,
-                    };
-                },
-                [&](DerivationOutputCAFloating dof) {
-                   return dof;
-                },
-            }, output.output);
+        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
             auto & st = outputStats.at(outputName);
             if (outputHash.method == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
@@ -4055,9 +4020,41 @@ void DerivationGoal::registerOutputs()
                 newInfo0.narSize = narHashAndSize.second;
             }
 
-            /* Check wanted hash if output is fixed */
-            if (auto p = std::get_if<DerivationOutputCAFixed>(&output.output)) {
-                Hash & wanted = p->hash.hash;
+            assert(newInfo0.ca);
+            return newInfo0;
+        };
+
+        ValidPathInfo newInfo = std::visit(overloaded {
+            [&](DerivationOutputInputAddressed output) {
+                /* input-addressed case */
+                auto requiredFinalPath = output.path;
+                /* Preemtively add rewrite rule for final hash, as that is
+                   what the NAR hash will use rather than normalized-self references */
+                if (scratchPath != requiredFinalPath)
+                    outputRewrites.insert_or_assign(
+                        std::string { scratchPath.hashPart() },
+                        std::string { requiredFinalPath.hashPart() });
+                rewriteOutput();
+                auto narHashAndSize = hashPath(htSHA256, actualPath);
+                ValidPathInfo newInfo0 { requiredFinalPath };
+                newInfo0.narHash = narHashAndSize.first;
+                newInfo0.narSize = narHashAndSize.second;
+                auto refs = rewriteRefs();
+                newInfo0.references = refs.second;
+                if (refs.first)
+                    newInfo0.references.insert(newInfo0.path);
+                return newInfo0;
+            },
+            [&](DerivationOutputCAFixed dof) {
+                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
+                    .method = dof.hash.method,
+                    .hashType = dof.hash.hash.type,
+                });
+
+                /* Check wanted hash */
+                Hash & wanted = dof.hash.hash;
+                assert(newInfo0.ca);
+                auto got = getContentAddressHash(*newInfo0.ca);
                 if (wanted != got) {
                     /* Throw an error after registering the path as
                        valid. */
@@ -4068,11 +4065,12 @@ void DerivationGoal::registerOutputs()
                             wanted.to_string(SRI, true),
                             got.to_string(SRI, true)));
                 }
-            }
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        }}();
+                return newInfo0;
+            },
+            [&](DerivationOutputCAFloating dof) {
+                return newInfoFromCA(dof);
+            },
+        }, output.output);
 
         /* Calculate where we'll move the output files. In the checking case we
            will leave leave them where they are, for now, rather than move to

From 8d4162ff9e940ea9e2f97b07f3030a722695901a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 12 Aug 2020 15:14:56 +0000
Subject: [PATCH 098/882] Separate auth and logic for the daemon

Before, processConnection wanted to know a user name and user id, and
`nix-daemon --stdio`, when it isn't proxying to an underlying daemon,
would just assume "root" and 0. But `nix-daemon --stdio` (no proxying)
shouldn't make guesses about who holds the other end of its standard
streams.

Now processConnection takes an "auth hook", so `nix-daemon` can provide
the appropriate policy and daemon.cc doesn't need to know or care what
it is.
---
 src/libstore/build.cc        |  3 ++-
 src/libstore/daemon.cc       | 13 ++-----------
 src/libstore/daemon.hh       |  7 +++++--
 src/nix-daemon/nix-daemon.cc | 15 +++++++++++++--
 tests/remote-store.sh        |  3 +++
 5 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 76baa1a6e..3fb052f00 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2920,7 +2920,8 @@ void DerivationGoal::startDaemon()
                 FdSink to(remote.get());
                 try {
                     daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive, "nobody", 65535);
+                        daemon::NotTrusted, daemon::Recursive,
+                        [&](Store & store) { store.createUser("nobody", 65535); });
                     debug("terminated daemon connection");
                 } catch (SysError &) {
                     ignoreException();
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 5e568fc94..7a6eb99be 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -817,8 +817,7 @@ void processConnection(
     FdSink & to,
     TrustedFlag trusted,
     RecursiveFlag recursive,
-    const std::string & userName,
-    uid_t userId)
+    std::function<void(Store &)> authHook)
 {
     auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
 
@@ -859,15 +858,7 @@ void processConnection(
 
         /* If we can't accept clientVersion, then throw an error
            *here* (not above). */
-
-#if 0
-        /* Prevent users from doing something very dangerous. */
-        if (geteuid() == 0 &&
-            querySetting("build-users-group", "") == "")
-            throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
-#endif
-
-        store->createUser(userName, userId);
+        authHook(*store);
 
         tunnelLogger->stopWork();
         to.flush();
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
index 266932013..841ace316 100644
--- a/src/libstore/daemon.hh
+++ b/src/libstore/daemon.hh
@@ -12,7 +12,10 @@ void processConnection(
     FdSink & to,
     TrustedFlag trusted,
     RecursiveFlag recursive,
-    const std::string & userName,
-    uid_t userId);
+    /* Arbitrary hook to check authorization / initialize user data / whatever
+       after the protocol has been negotiated. The idea is that this function
+       and everything it calls doesn't know about this stuff, and the
+       `nix-daemon` handles that instead. */
+    std::function<void(Store &)> authHook);
 
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index bcb86cbce..cfa634a44 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -239,7 +239,15 @@ static void daemonLoop(char * * argv)
                 //  Handle the connection.
                 FdSource from(remote.get());
                 FdSink to(remote.get());
-                processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid);
+                processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) {
+#if 0
+                    /* Prevent users from doing something very dangerous. */
+                    if (geteuid() == 0 &&
+                        querySetting("build-users-group", "") == "")
+                        throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
+#endif
+                    store.createUser(user, peer.uid);
+                });
 
                 exit(0);
             }, options);
@@ -324,7 +332,10 @@ static int _main(int argc, char * * argv)
             } else {
                 FdSource from(STDIN_FILENO);
                 FdSink to(STDOUT_FILENO);
-                processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0);
+                /* Auth hook is empty because in this mode we blindly trust the
+                   standard streams. Limitting access to thoses is explicitly
+                   not `nix-daemon`'s responsibility. */
+                processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
             }
         } else {
             daemonLoop(argv);
diff --git a/tests/remote-store.sh b/tests/remote-store.sh
index 4cc73465a..3a61946f9 100644
--- a/tests/remote-store.sh
+++ b/tests/remote-store.sh
@@ -2,6 +2,9 @@ source common.sh
 
 clearStore
 
+# Ensure "fake ssh" remote store works just as legacy fake ssh would.
+nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor
+
 startDaemon
 
 storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh

From 4720853129b6866775edd9f90ad6f10701f98a3c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 12 Aug 2020 16:32:36 +0000
Subject: [PATCH 099/882] Make `system-features` a store setting

This seems more correct. It also means one can specify the features a
store should support with --store and remote-store=..., which is useful.
I use this to clean up the build remotes test.
---
 src/build-remote/build-remote.cc   | 16 ++++-----------
 src/libstore/build.cc              |  8 ++++----
 src/libstore/machines.cc           | 24 ++++++++++++++++++++++
 src/libstore/machines.hh           |  4 ++++
 src/libstore/parsed-derivations.cc |  8 ++++----
 src/libstore/parsed-derivations.hh |  4 ++--
 src/libstore/store-api.hh          |  4 ++++
 tests/build-hook.nix               |  3 ++-
 tests/build-remote.sh              | 33 +++++++++++++++++-------------
 tests/local.mk                     |  2 +-
 10 files changed, 68 insertions(+), 38 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index cec7b369b..ce5127113 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -38,9 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
+static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
     for (auto & feature : requiredFeatures)
-        if (!settings.systemFeatures.get().count(feature)) return false;
+        if (!store.systemFeatures.get().count(feature)) return false;
     return true;
 }
 
@@ -106,7 +106,7 @@ static int _main(int argc, char * * argv)
             auto canBuildLocally = amWilling
                  &&  (  neededSystem == settings.thisSystem
                      || settings.extraPlatforms.get().count(neededSystem) > 0)
-                 &&  allSupportedLocally(requiredFeatures);
+                 &&  allSupportedLocally(*store, requiredFeatures);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -224,15 +224,7 @@ static int _main(int argc, char * * argv)
 
                     Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
 
-                    Store::Params storeParams;
-                    if (hasPrefix(bestMachine->storeUri, "ssh://")) {
-                        storeParams["max-connections"] = "1";
-                        storeParams["log-fd"] = "4";
-                        if (bestMachine->sshKey != "")
-                            storeParams["ssh-key"] = bestMachine->sshKey;
-                    }
-
-                    sshStore = openStore(bestMachine->storeUri, storeParams);
+                    sshStore = bestMachine->openStore();
                     sshStore->connect();
                     storeUri = bestMachine->storeUri;
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 76baa1a6e..d6e6ad6a9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1475,7 +1475,7 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
+    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
 
     /* Is the build hook willing to accept this job? */
     if (!buildLocally) {
@@ -1964,13 +1964,13 @@ void linkOrCopy(const Path & from, const Path & to)
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
-    if (!parsedDrv->canBuildLocally())
+    if (!parsedDrv->canBuildLocally(worker.store))
         throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
             drv->platform,
             concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
             worker.store.printStorePath(drvPath),
             settings.thisSystem,
-            concatStringsSep<StringSet>(", ", settings.systemFeatures));
+            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
 
     if (drv->isBuiltin())
         preloadNSS();
@@ -3179,7 +3179,7 @@ void DerivationGoal::runChild()
                 createDirs(chrootRootDir + "/dev/shm");
                 createDirs(chrootRootDir + "/dev/pts");
                 ss.push_back("/dev/full");
-                if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
+                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
                     ss.push_back("/dev/kvm");
                 ss.push_back("/dev/null");
                 ss.push_back("/dev/random");
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index f848582da..7db2556f4 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -1,6 +1,7 @@
 #include "machines.hh"
 #include "util.hh"
 #include "globals.hh"
+#include "store-api.hh"
 
 #include <algorithm>
 
@@ -48,6 +49,29 @@ bool Machine::mandatoryMet(const std::set<string> & features) const {
         });
 }
 
+ref<Store> Machine::openStore() const {
+    Store::Params storeParams;
+    if (hasPrefix(storeUri, "ssh://")) {
+        storeParams["max-connections"] = "1";
+        storeParams["log-fd"] = "4";
+        if (sshKey != "")
+            storeParams["ssh-key"] = sshKey;
+    }
+    {
+        auto & fs = storeParams["system-features"];
+        auto append = [&](auto feats) {
+            for (auto & f : feats) {
+                if (fs.size() > 0) fs += ' ';
+                fs += f;
+            }
+        };
+        append(supportedFeatures);
+        append(mandatoryFeatures);
+    }
+
+    return nix::openStore(storeUri, storeParams);
+}
+
 void parseMachines(const std::string & s, Machines & machines)
 {
     for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh
index de92eb924..341d9bd97 100644
--- a/src/libstore/machines.hh
+++ b/src/libstore/machines.hh
@@ -4,6 +4,8 @@
 
 namespace nix {
 
+class Store;
+
 struct Machine {
 
     const string storeUri;
@@ -28,6 +30,8 @@ struct Machine {
         decltype(supportedFeatures) supportedFeatures,
         decltype(mandatoryFeatures) mandatoryFeatures,
         decltype(sshPublicHostKey) sshPublicHostKey);
+
+    ref<Store> openStore() const;
 };
 
 typedef std::vector<Machine> Machines;
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 24f848e46..e7b7202d4 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -94,7 +94,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
     return res;
 }
 
-bool ParsedDerivation::canBuildLocally() const
+bool ParsedDerivation::canBuildLocally(Store & localStore) const
 {
     if (drv.platform != settings.thisSystem.get()
         && !settings.extraPlatforms.get().count(drv.platform)
@@ -102,14 +102,14 @@ bool ParsedDerivation::canBuildLocally() const
         return false;
 
     for (auto & feature : getRequiredSystemFeatures())
-        if (!settings.systemFeatures.get().count(feature)) return false;
+        if (!localStore.systemFeatures.get().count(feature)) return false;
 
     return true;
 }
 
-bool ParsedDerivation::willBuildLocally() const
+bool ParsedDerivation::willBuildLocally(Store & localStore) const
 {
-    return getBoolAttr("preferLocalBuild") && canBuildLocally();
+    return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore);
 }
 
 bool ParsedDerivation::substitutesAllowed() const
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 6ee172d81..3fa09f34f 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -29,9 +29,9 @@ public:
 
     StringSet getRequiredSystemFeatures() const;
 
-    bool canBuildLocally() const;
+    bool canBuildLocally(Store & localStore) const;
 
-    bool willBuildLocally() const;
+    bool willBuildLocally(Store & localStore) const;
 
     bool substitutesAllowed() const;
 };
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e94d975c5..1e940e6a8 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -164,6 +164,10 @@ public:
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
 
+    Setting<StringSet> systemFeatures{this, settings.systemFeatures,
+        "system-features",
+        "Optional features that the system this store builds on implements (like \"kvm\")."};
+
 protected:
 
     struct PathInfoCacheValue {
diff --git a/tests/build-hook.nix b/tests/build-hook.nix
index a19c10dde..1bd0b759f 100644
--- a/tests/build-hook.nix
+++ b/tests/build-hook.nix
@@ -23,6 +23,7 @@ let
     shell = busybox;
     name = "build-remote-input-2";
     buildCommand = "echo BAR > $out";
+    requiredSystemFeatures = ["bar"];
   };
 
 in
@@ -34,6 +35,6 @@ in
       ''
         read x < ${input1}
         read y < ${input2}
-        echo $x$y > $out
+        echo "$x $y" > $out
       '';
   }
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 4dfb753e1..7638f536f 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -1,31 +1,36 @@
 source common.sh
 
-clearStore
-
 if ! canUseSandbox; then exit; fi
 if ! [[ $busybox =~ busybox ]]; then exit; fi
 
-chmod -R u+w $TEST_ROOT/machine0 || true
-chmod -R u+w $TEST_ROOT/machine1 || true
-chmod -R u+w $TEST_ROOT/machine2 || true
-rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2
-rm -f $TEST_ROOT/result
-
 unset NIX_STORE_DIR
 unset NIX_STATE_DIR
 
+function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
+
+builders=(
+  # system-features will automatically be added to the outer URL, but not inner
+  # remote-store URL.
+  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
+  "$TEST_ROOT/machine2 - - 1 1 bar"
+)
+
 # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
 # child process. This allows us to test LegacySSHStore::buildDerivation().
+# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().
 nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
   --arg busybox $busybox \
   --store $TEST_ROOT/machine0 \
-  --builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \
-  --system-features foo
+  --builders "$(join_by '; ' "${builders[@]}")"
 
 outPath=$(readlink -f $TEST_ROOT/result)
 
-cat $TEST_ROOT/machine0/$outPath | grep FOOBAR
+grep 'FOO BAR' $TEST_ROOT/machine0/$outPath
 
-# Ensure that input1 was built on store2 due to the required feature.
-(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh)
-nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh
+# Ensure that input1 was built on store1 due to the required feature.
+(! nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh)
+nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh
+
+# Ensure that input2 was built on store2 due to the required feature.
+(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-2.sh)
+nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-2.sh
diff --git a/tests/local.mk b/tests/local.mk
index 0f3bfe606..5c77b9bb7 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -1,5 +1,5 @@
 nix_tests = \
-  init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
+  hash.sh lang.sh add.sh simple.sh dependencies.sh \
   config.sh \
   gc.sh \
   gc-concurrent.sh \

From d2f2be0f701f8b091a00b8898dc7fb922096cfaf Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 12 Aug 2020 13:24:39 +0000
Subject: [PATCH 100/882] Test `RemoteStore::buildDerivation`

Fix `wopNarFromPath` which needed a `toRealPath`.
---
 src/libstore/daemon.cc |  2 +-
 tests/build-hook.nix   | 12 +++++++++++-
 tests/build-remote.sh  | 23 ++++++++++++++++++-----
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 7a6eb99be..956a8dc8d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -688,7 +688,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->printStorePath(path), to);
+        dumpPath(store->toRealPath(store->printStorePath(path)), to);
         break;
     }
 
diff --git a/tests/build-hook.nix b/tests/build-hook.nix
index 1bd0b759f..eb16676f0 100644
--- a/tests/build-hook.nix
+++ b/tests/build-hook.nix
@@ -26,6 +26,16 @@ let
     requiredSystemFeatures = ["bar"];
   };
 
+  input3 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-3";
+    buildCommand = ''
+      read x < ${input2}
+      echo $x BAZ > $out
+    '';
+    requiredSystemFeatures = ["baz"];
+  };
+
 in
 
   mkDerivation {
@@ -34,7 +44,7 @@ in
     buildCommand =
       ''
         read x < ${input1}
-        read y < ${input2}
+        read y < ${input3}
         echo "$x $y" > $out
       '';
   }
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 7638f536f..8833f4698 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -13,6 +13,7 @@ builders=(
   # remote-store URL.
   "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
   "$TEST_ROOT/machine2 - - 1 1 bar"
+  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
 )
 
 # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
@@ -25,12 +26,24 @@ nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
 
 outPath=$(readlink -f $TEST_ROOT/result)
 
-grep 'FOO BAR' $TEST_ROOT/machine0/$outPath
+grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath
+
+set -o pipefail
 
 # Ensure that input1 was built on store1 due to the required feature.
-(! nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh)
-nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh
+nix path-info --store $TEST_ROOT/machine1 --all \
+  | grep builder-build-remote-input-1.sh \
+  | grep -v builder-build-remote-input-2.sh \
+  | grep -v builder-build-remote-input-3.sh
 
 # Ensure that input2 was built on store2 due to the required feature.
-(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-2.sh)
-nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-2.sh
+nix path-info --store $TEST_ROOT/machine2 --all \
+  | grep -v builder-build-remote-input-1.sh \
+  | grep builder-build-remote-input-2.sh \
+  | grep -v builder-build-remote-input-3.sh
+
+# Ensure that input3 was built on store3 due to the required feature.
+nix path-info --store $TEST_ROOT/machine3 --all \
+  | grep -v builder-build-remote-input-1.sh \
+  | grep -v builder-build-remote-input-2.sh \
+  | grep builder-build-remote-input-3.sh

From 85aacbee64680e60b22e8a729756b0a3882f8b8f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Aug 2020 14:47:53 +0000
Subject: [PATCH 101/882] Use `TeeSink` and `TeeSouce` in a few more places

---
 src/libstore/binary-cache-store.cc | 12 ++++--------
 src/libstore/local-store.cc        |  6 +-----
 src/libstore/store-api.cc          |  6 +++---
 src/libutil/archive.cc             |  6 +-----
 src/libutil/serialise.hh           | 11 +++++++++++
 5 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 86877dd1a..c360b9dda 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -312,14 +312,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    uint64_t narSize = 0;
+    LengthSink narSize;
+    TeeSink tee { sink, narSize };
 
-    LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
-        sink(data, len);
-        narSize += len;
-    });
-
-    auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
+    auto decompressor = makeDecompressionSink(info->compression, tee);
 
     try {
         getFile(info->url, *decompressor);
@@ -331,7 +327,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 
     stats.narRead++;
     //stats.narReadCompressedBytes += nar->size(); // FIXME
-    stats.narReadBytes += narSize;
+    stats.narReadBytes += narSize.length;
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 3c66a4dfd..e552bdc59 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1021,11 +1021,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             else
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
 
-            LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
-                size_t n = source.read(data, len);
-                (*hashSink)(data, n);
-                return n;
-            });
+            TeeSource wrapperSource { source, *hashSink };
 
             restorePath(realPath, wrapperSource);
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 2837910b9..3d07e2d38 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -746,12 +746,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     }
 
     auto source = sinkToSource([&](Sink & sink) {
-        LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
-            sink(data, len);
+        LambdaSink progressSink([&](const unsigned char * data, size_t len) {
             total += len;
             act.progress(total, info->narSize);
         });
-        srcStore->narFromPath(storePath, wrapperSink);
+        TeeSink tee { sink, progressSink };
+        srcStore->narFromPath(storePath, tee);
     }, [&]() {
            throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri());
     });
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index ce7cf9754..14399dea3 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -366,11 +366,7 @@ void copyNAR(Source & source, Sink & sink)
 
     ParseSink parseSink; /* null sink; just parse the NAR */
 
-    LambdaSource wrapper([&](unsigned char * data, size_t len) {
-        auto n = source.read(data, len);
-        sink(data, n);
-        return n;
-    });
+    TeeSource wrapper { source, sink };
 
     parseDump(parseSink, wrapper);
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index c29c6b29b..69ae0874a 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -225,6 +225,17 @@ struct SizedSource : Source
     }
 };
 
+/* A sink that that just counts the number of bytes given to it */
+struct LengthSink : Sink
+{
+    uint64_t length = 0;
+
+    virtual void operator () (const unsigned char * _, size_t len)
+    {
+        length += len;
+    }
+};
+
 /* Convert a function into a sink. */
 struct LambdaSink : Sink
 {

From 5ccd94501dac232cc09fb5301c4406cef72c0a27 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Aug 2020 13:43:29 +0000
Subject: [PATCH 102/882] Allow trustless building of CA derivations

Include a long comment explaining the policy. Perhaps this can be moved
to the manual at some point in the future.

Also bump the daemon protocol minor version, so clients can tell whether
`wopBuildDerivation` supports trustless CA derivation building. I hope
to take advantage of this in a follow-up PR to support trustless remote
building with the minimal sending of derivation closures.
---
 src/libstore/daemon.cc          | 42 +++++++++++++++++++++++++++++++--
 src/libstore/worker-protocol.hh |  2 +-
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 956a8dc8d..45e81c8da 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -454,8 +454,46 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
         BuildMode buildMode = (BuildMode) readInt(from);
         logger->startWork();
-        if (!trusted)
-            throw Error("you are not privileged to build derivations");
+
+        /* Content-addressed derivations are trustless because their output paths
+           are verified by their content alone, so any derivation is free to
+           try to produce such a path.
+
+           Input-addressed derivation output paths, however, are calculated
+           from the derivation closure that produced them---even knowing the
+           root derivation is not enough. That the output data actually came
+           from those derivations is fundamentally unverifiable, but the daemon
+           trusts itself on that matter. The question instead is whether the
+           submitted plan has rights to the output paths it wants to fill, and
+           at least the derivation closure proves that.
+
+           It would have been nice if input-address algorithm merely depended
+           on the build time closure, rather than depending on the derivation
+           closure. That would mean input-addressed paths used at build time
+           would just be trusted and not need their own evidence. This is in
+           fact fine as the same guarantees would hold *inductively*: either
+           the remote builder has those paths and already trusts them, or it
+           needs to build them too and thus their evidence must be provided in
+           turn.  The advantage of this variant algorithm is that the evidence
+           for input-addressed paths which the remote builder already has
+           doesn't need to be sent again.
+
+           That said, now that we have floating CA derivations, it is better
+           that people just migrate to those which also solve this problem, and
+           others. It's the same migration difficulty with strictly more
+           benefit.
+
+           Lastly, do note that when we parse fixed-output content-addressed
+           derivations, we throw out the precomputed output paths and just
+           store the hashes, so there aren't two competing sources of truth an
+           attacker could exploit. */
+        if (drv.type() == DerivationType::InputAddressed && !trusted)
+            throw Error("you are not privileged to build input-addressed derivations");
+
+        /* Make sure that the non-input-addressed derivations that got this far
+           are in fact content-addressed if we don't trust them. */
+        assert(derivationIsCA(drv.type()) || trusted);
+
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
         to << res.status << res.errorMsg;
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index f76b13fb4..5eddaff56 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x117
+#define PROTOCOL_VERSION 0x118
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

From e1308b121169ea8327c95556668ad4f7f4815402 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 12 Aug 2020 03:13:17 +0000
Subject: [PATCH 103/882] Define `LegacySSHStore::buildPaths` using
 `cmdBuildPaths`

Evidentally this was never implemented because Nix switched to using
`buildDerivation` exclusively before `build-remote.pl` was rewritten.

The `nix-copy-ssh` test (already) tests this.
---
 src/libstore/legacy-ssh-store.cc | 53 ++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 10 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index c6eeab548..b5ece22f4 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -202,6 +202,24 @@ struct LegacySSHStore : public Store
         const StorePathSet & references, RepairFlag repair) override
     { unsupported("addTextToStore"); }
 
+private:
+
+    void putBuildSettings(Connection & conn)
+    {
+        conn.to
+            << settings.maxSilentTime
+            << settings.buildTimeout;
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
+            conn.to
+                << settings.maxLogSize;
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3)
+            conn.to
+                << settings.buildRepeat
+                << settings.enforceDeterminism;
+    }
+
+public:
+
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     {
@@ -211,16 +229,8 @@ struct LegacySSHStore : public Store
             << cmdBuildDerivation
             << printStorePath(drvPath);
         writeDerivation(conn->to, *this, drv);
-        conn->to
-            << settings.maxSilentTime
-            << settings.buildTimeout;
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
-            conn->to
-                << settings.maxLogSize;
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
-            conn->to
-                << settings.buildRepeat
-                << settings.enforceDeterminism;
+
+        putBuildSettings(*conn);
 
         conn->to.flush();
 
@@ -234,6 +244,29 @@ struct LegacySSHStore : public Store
         return status;
     }
 
+    void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override
+    {
+        auto conn(connections->get());
+
+        conn->to << cmdBuildPaths;
+        Strings ss;
+        for (auto & p : drvPaths)
+            ss.push_back(p.to_string(*this));
+        conn->to << ss;
+
+        putBuildSettings(*conn);
+
+        conn->to.flush();
+
+        BuildResult result;
+        result.status = (BuildResult::Status) readInt(conn->from);
+
+        if (!result.success()) {
+            conn->from >> result.errorMsg;
+            throw Error(result.status, result.errorMsg);
+        }
+    }
+
     void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 

From ed026f7206a3154ce11bddac2e58541327313f6f Mon Sep 17 00:00:00 2001
From: Chuck <chuck@intelligence.org>
Date: Thu, 13 Aug 2020 17:44:42 -0700
Subject: [PATCH 104/882] Don't try to parse signature check as commit
 timestamp

When the log.showSignature git setting is enabled, the output of
"git log" contains signature verification information in addition to the
timestamp GitInputScheme::fetch wants:

  $ git log -1 --format=%ct
  gpg: Signature made Sat 07 Sep 2019 02:02:03 PM PDT
  gpg:                using RSA key 0123456789ABCDEF0123456789ABCDEF01234567
  gpg:                issuer "user@example.com"
  gpg: Good signature from "User <user@example.com>" [ultimate] 1567890123
  1567890123

For folks that had log.showSignature set, this caused all nix operations
on flakes to fail:

  $ nix build
  error: stoull
---
 src/libfetchers/git.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 8b6e047f1..5ca0f8521 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -269,7 +269,7 @@ struct GitInputScheme : InputScheme
                 // modified dirty file?
                 input.attrs.insert_or_assign(
                     "lastModified",
-                    haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
+                    haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
 
                 return {
                     Tree(store->printStorePath(storePath), std::move(storePath)),
@@ -421,7 +421,7 @@ struct GitInputScheme : InputScheme
 
         auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},

From 4b571ea3216715ac1f2c06d1b0d68f27c6070d28 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 14 Aug 2020 11:52:37 -0400
Subject: [PATCH 105/882] Update src/libstore/daemon.cc

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 956a8dc8d..80ed64f02 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -688,7 +688,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->toRealPath(store->printStorePath(path)), to);
+        dumpPath(store->toRealPath(path)), to);
         break;
     }
 

From f899a7c6d726dbf75e78fce5b5a9aa478387fcbe Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 14 Aug 2020 18:51:31 +0000
Subject: [PATCH 106/882] Work around clang bug

---
 src/nix/show-derivation.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index cc52e53fb..b9f33499b 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -67,7 +67,8 @@ struct CmdShowDerivation : InstallablesCommand
 
             {
                 auto outputsObj(drvObj.object("outputs"));
-                for (auto & [outputName, output] : drv.outputs) {
+                for (auto & [_outputName, output] : drv.outputs) {
+                    auto & outputName = _outputName; // work around clang bug
                     auto outputObj { outputsObj.object(outputName) };
                     std::visit(overloaded {
                         [&](DerivationOutputInputAddressed doi) {

From 6f7ac5e8658aad4c0a44c220d9a2b06ea4564980 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 14 Aug 2020 21:59:31 +0000
Subject: [PATCH 107/882] Remove extra closing paren

---
 src/libstore/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 80ed64f02..dde4122d1 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -688,7 +688,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->toRealPath(path)), to);
+        dumpPath(store->toRealPath(path), to);
         break;
     }
 

From dbf96e10ecc75410c9db798f208f8a8310842a4f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 16 Aug 2020 17:38:12 +0000
Subject: [PATCH 108/882] Test remote building with fixed output derivations

---
 tests/build-hook-ca.nix                       | 45 +++++++++++++++++++
 tests/build-remote-content-addressed-fixed.sh |  5 +++
 tests/build-remote-input-addressed.sh         |  5 +++
 tests/build-remote.sh                         |  4 +-
 tests/local.mk                                |  3 +-
 5 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 tests/build-hook-ca.nix
 create mode 100644 tests/build-remote-content-addressed-fixed.sh
 create mode 100644 tests/build-remote-input-addressed.sh

diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca.nix
new file mode 100644
index 000000000..98db473fc
--- /dev/null
+++ b/tests/build-hook-ca.nix
@@ -0,0 +1,45 @@
+{ busybox }:
+
+with import ./config.nix;
+
+let
+
+  mkDerivation = args:
+    derivation ({
+      inherit system;
+      builder = busybox;
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      outputHashMode = "recursive";
+      outputHashAlgo = "sha256";
+    } // removeAttrs args ["builder" "meta"])
+    // { meta = args.meta or {}; };
+
+  input1 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-1";
+    buildCommand = "echo FOO > $out";
+    requiredSystemFeatures = ["foo"];
+    outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=";
+  };
+
+  input2 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-2";
+    buildCommand = "echo BAR > $out";
+    requiredSystemFeatures = ["bar"];
+    outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q=";
+  };
+
+in
+
+  mkDerivation {
+    shell = busybox;
+    name = "build-remote";
+    buildCommand =
+      ''
+        read x < ${input1}
+        read y < ${input2}
+        echo "$x $y" > $out
+      '';
+    outputHash = "sha256-3YGhlOfbGUm9hiPn2teXXTT8M1NEpDFvfXkxMaJRld0=";
+  }
diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh
new file mode 100644
index 000000000..1408a19d5
--- /dev/null
+++ b/tests/build-remote-content-addressed-fixed.sh
@@ -0,0 +1,5 @@
+source common.sh
+
+file=build-hook-ca.nix
+
+source build-remote.sh
diff --git a/tests/build-remote-input-addressed.sh b/tests/build-remote-input-addressed.sh
new file mode 100644
index 000000000..b34caa061
--- /dev/null
+++ b/tests/build-remote-input-addressed.sh
@@ -0,0 +1,5 @@
+source common.sh
+
+file=build-hook.nix
+
+source build-remote.sh
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 7638f536f..d9048583f 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -1,5 +1,3 @@
-source common.sh
-
 if ! canUseSandbox; then exit; fi
 if ! [[ $busybox =~ busybox ]]; then exit; fi
 
@@ -18,7 +16,7 @@ builders=(
 # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
 # child process. This allows us to test LegacySSHStore::buildDerivation().
 # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().
-nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
+nix build -L -v -f $file -o $TEST_ROOT/result --max-jobs 0 \
   --arg busybox $busybox \
   --store $TEST_ROOT/machine0 \
   --builders "$(join_by '; ' "${builders[@]}")"
diff --git a/tests/local.mk b/tests/local.mk
index 5c77b9bb7..492d6a0fd 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -14,7 +14,8 @@ nix_tests = \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \
   build-dry.sh \
-  build-remote.sh \
+  build-remote-input-addressed.sh \
+  build-remote-content-addressed-fixed.sh \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \

From a72a20d68fe6ae75f05f69e7de0bc3326d779144 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Aug 2020 17:44:52 +0200
Subject: [PATCH 109/882] Add 'nix dump-args' to dump all commands/flags for
 manpage generation

---
 src/libutil/args.cc | 70 +++++++++++++++++++++++++++++++++++++++++++++
 src/libutil/args.hh |  8 ++++++
 src/nix/command.cc  | 13 +++++++++
 src/nix/command.hh  |  7 +++++
 src/nix/flake.cc    |  7 +----
 src/nix/main.cc     |  7 +++++
 src/nix/profile.cc  |  7 +----
 src/nix/registry.cc |  7 +----
 8 files changed, 108 insertions(+), 18 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 986c5d1cd..bf5e7ce95 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -3,6 +3,8 @@
 
 #include <glob.h>
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 void Args::addFlag(Flag && flag_)
@@ -205,6 +207,42 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
+nlohmann::json Args::toJSON()
+{
+    auto flags = nlohmann::json::object();
+
+    for (auto & [name, flag] : longFlags) {
+        auto j = nlohmann::json::object();
+        if (flag->shortName)
+            j["shortName"] = std::string(1, flag->shortName);
+        if (flag->description != "")
+            j["description"] = flag->description;
+        if (flag->category != "")
+            j["category"] = flag->category;
+        if (flag->handler.arity != ArityAny)
+            j["arity"] = flag->handler.arity;
+        if (!flag->labels.empty())
+            j["labels"] = flag->labels;
+        flags[name] = std::move(j);
+    }
+
+    auto args = nlohmann::json::array();
+
+    for (auto & arg : expectedArgs) {
+        auto j = nlohmann::json::object();
+        j["label"] = arg.label;
+        j["optional"] = arg.optional;
+        if (arg.handler.arity != ArityAny)
+            j["arity"] = arg.handler.arity;
+        args.push_back(std::move(j));
+    }
+
+    auto res = nlohmann::json::object();
+    res["flags"] = std::move(flags);
+    res["args"] = std::move(args);
+    return res;
+}
+
 static void hashTypeCompleter(size_t index, std::string_view prefix) 
 {
     for (auto & type : hashTypes)
@@ -313,6 +351,22 @@ void Command::printHelp(const string & programName, std::ostream & out)
     }
 }
 
+nlohmann::json Command::toJSON()
+{
+    auto exs = nlohmann::json::array();
+
+    for (auto & example : examples()) {
+        auto ex = nlohmann::json::object();
+        ex["description"] = example.description;
+        ex["command"] = example.command;
+        exs.push_back(std::move(ex));
+    }
+
+    auto res = Args::toJSON();
+    res["examples"] = std::move(exs);
+    return res;
+}
+
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
@@ -387,4 +441,20 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+nlohmann::json MultiCommand::toJSON()
+{
+    auto cmds = nlohmann::json::object();
+
+    for (auto & [name, commandFun] : commands) {
+        auto command = commandFun();
+        auto j = command->toJSON();
+        j["category"] = categories[command->category()];
+        cmds[name] = std::move(j);
+    }
+
+    auto res = Args::toJSON();
+    res["commands"] = std::move(cmds);
+    return res;
+}
+
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 97a517344..c56044f1d 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -4,6 +4,8 @@
 #include <map>
 #include <memory>
 
+#include <nlohmann/json_fwd.hpp>
+
 #include "util.hh"
 
 namespace nix {
@@ -203,6 +205,8 @@ public:
         });
     }
 
+    virtual nlohmann::json toJSON();
+
     friend class MultiCommand;
 };
 
@@ -234,6 +238,8 @@ struct Command : virtual Args
     virtual Category category() { return catDefault; }
 
     void printHelp(const string & programName, std::ostream & out) override;
+
+    nlohmann::json toJSON() override;
 };
 
 typedef std::map<std::string, std::function<ref<Command>()>> Commands;
@@ -259,6 +265,8 @@ public:
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
+
+    nlohmann::json toJSON() override;
 };
 
 Strings argvToStrings(int argc, char * * argv);
diff --git a/src/nix/command.cc b/src/nix/command.cc
index da32819da..2f1fbded1 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -4,12 +4,25 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 
+#include <nlohmann/json.hpp>
+
 extern char * * environ __attribute__((weak));
 
 namespace nix {
 
 Commands * RegisterCommand::commands = nullptr;
 
+void NixMultiCommand::printHelp(const string & programName, std::ostream & out)
+{
+    MultiCommand::printHelp(programName, out);
+}
+
+nlohmann::json NixMultiCommand::toJSON()
+{
+    // FIXME: use Command::toJSON() as well.
+    return MultiCommand::toJSON();
+}
+
 StoreCommand::StoreCommand()
 {
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index bc46a2028..d60c8aeb6 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -21,6 +21,13 @@ static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;
 
+struct NixMultiCommand : virtual MultiCommand, virtual Command
+{
+    void printHelp(const string & programName, std::ostream & out) override;
+
+    nlohmann::json toJSON() override;
+};
+
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 653f8db1b..ae6f4c5f9 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -933,7 +933,7 @@ struct CmdFlakeShow : FlakeCommand
     }
 };
 
-struct CmdFlake : virtual MultiCommand, virtual Command
+struct CmdFlake : NixMultiCommand
 {
     CmdFlake()
         : MultiCommand({
@@ -963,11 +963,6 @@ struct CmdFlake : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdFlake>("flake");
diff --git a/src/nix/main.cc b/src/nix/main.cc
index e62657e95..fa04bebea 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -17,6 +17,8 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
+#include <nlohmann/json.hpp>
+
 extern std::string chrootHelperName;
 
 void chrootHelper(int argc, char * * argv);
@@ -172,6 +174,11 @@ void mainWrapped(int argc, char * * argv)
 
     NixArgs args;
 
+    if (argc == 2 && std::string(argv[1]) == "dump-args") {
+        std::cout << args.toJSON().dump() << "\n";
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index cffc9ee44..9241931e9 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -437,7 +437,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
     }
 };
 
-struct CmdProfile : virtual MultiCommand, virtual Command
+struct CmdProfile : NixMultiCommand
 {
     CmdProfile()
         : MultiCommand({
@@ -461,11 +461,6 @@ struct CmdProfile : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdProfile>("profile");
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index ebee4545c..367268683 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -115,7 +115,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand
     }
 };
 
-struct CmdRegistry : virtual MultiCommand, virtual Command
+struct CmdRegistry : virtual NixMultiCommand
 {
     CmdRegistry()
         : MultiCommand({
@@ -141,11 +141,6 @@ struct CmdRegistry : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdRegistry>("registry");

From 6f19c776db280a46e84d6e84d83814445869ef37 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Aug 2020 19:33:18 +0200
Subject: [PATCH 110/882] Start generation of the nix.1 manpage

---
 .gitignore                                    |  2 +
 doc/manual/generate-manpage.jq                | 42 +++++++++++++++++++
 doc/manual/local.mk                           |  8 +++-
 doc/manual/src/SUMMARY.md                     |  2 +
 .../src/command-ref/experimental-commands.md  |  8 ++++
 src/libutil/args.cc                           |  3 +-
 6 files changed, 63 insertions(+), 2 deletions(-)
 create mode 100644 doc/manual/generate-manpage.jq
 create mode 100644 doc/manual/src/command-ref/experimental-commands.md

diff --git a/.gitignore b/.gitignore
index d456d7da3..b8028e665 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,8 @@ perl/Makefile.config
 /doc/manual/*.1
 /doc/manual/*.5
 /doc/manual/*.8
+/doc/manual/nix.json
+/doc/manual/src/command-ref/nix.md
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/generate-manpage.jq b/doc/manual/generate-manpage.jq
new file mode 100644
index 000000000..cf06da0c8
--- /dev/null
+++ b/doc/manual/generate-manpage.jq
@@ -0,0 +1,42 @@
+def show_flags:
+
+.flags
+| map_values(select(.category != "config"))
+| to_entries
+| map(
+    "  - `--" + .key + "`"
+    + (if .value.shortName then " / `" + .value.shortName + "`" else "" end)
+    + (if .value.labels then " " + (.value.labels | map("*" + . + "*") | join(" ")) else "" end)
+    + "  \n"
+    + "    " + .value.description + "\n\n")
+| join("")
+;
+
+def show_synopsis:
+
+"`" + .command + "` " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n"
+
+;
+
+"# Synopsis\n\n"
++ ({"command": "nix", "args": .args} | show_synopsis)
++ "\n"
++ "# Common flags\n\n"
++ show_flags
++ (.commands | to_entries | map(
+    "# Operation `" + .key + "`\n\n"
+    + "## Synopsis\n\n"
+    + ({"command": ("nix " + .key), "args": .value.args} | show_synopsis)
+    + "\n"
+    + "## Description\n\n"
+    + .value.description + "\n\n"
+    + (if (.value.flags | length) > 0 then
+        "## Flags\n\n"
+        + (.value | show_flags)
+      else "" end)
+    + (if (.value.examples | length) > 0 then
+        "## Examples\n\n"
+        + (.value.examples | map("- " + .description + "\n\n  ```console\n  " + .command + "\n  ```\n" ) | join("\n"))
+        + "\n"
+       else "" end)
+  ) | join(""))
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index c9104ad7e..8f917316d 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -4,7 +4,7 @@ MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
 
 # Generate man pages.
 man-pages := $(foreach n, \
-  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
+  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 nix.1 \
   nix-collect-garbage.1 \
   nix-prefetch-url.1 nix-channel.1 \
   nix-hash.1 nix-copy-closure.1 \
@@ -24,6 +24,12 @@ $(d)/%.8: $(d)/src/command-ref/%.md
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
+$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.jq
+	jq -r -f doc/manual/generate-manpage.jq $< > $@
+
+$(d)/nix.json: $(bindir)/nix
+	$(trace-gen) $(bindir)/nix dump-args > $@
+
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 5f6817674..4089caf8a 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -60,6 +60,8 @@
     - [nix-hash](command-ref/nix-hash.md)
     - [nix-instantiate](command-ref/nix-instantiate.md)
     - [nix-prefetch-url](command-ref/nix-prefetch-url.md)
+  - [Experimental Commands](command-ref/experimental-commands.md)
+    - [nix](command-ref/nix.md)
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
 - [Glossary](glossary.md)
diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md
new file mode 100644
index 000000000..cfa6f8b73
--- /dev/null
+++ b/doc/manual/src/command-ref/experimental-commands.md
@@ -0,0 +1,8 @@
+# Experimental Commands
+
+This section lists experimental commands.
+
+> **Warning**
+>
+> These commands may be removed in the future, or their syntax may
+> change in incompatible ways.
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index bf5e7ce95..ad83a2414 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -238,6 +238,7 @@ nlohmann::json Args::toJSON()
     }
 
     auto res = nlohmann::json::object();
+    res["description"] = description();
     res["flags"] = std::move(flags);
     res["args"] = std::move(args);
     return res;
@@ -371,7 +372,7 @@ MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
     expectArgs({
-        .label = "command",
+        .label = "subcommand",
         .optional = true,
         .handler = {[=](std::string s) {
             assert(!command);

From 07975979aae4e7729ae13ffeb7390d07d71ad4bd Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Mon, 17 Aug 2020 15:04:54 -0400
Subject: [PATCH 111/882] Comment out fixed content address test

---
 tests/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/local.mk b/tests/local.mk
index 492d6a0fd..53035da41 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -15,7 +15,6 @@ nix_tests = \
   linux-sandbox.sh \
   build-dry.sh \
   build-remote-input-addressed.sh \
-  build-remote-content-addressed-fixed.sh \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
@@ -35,6 +34,7 @@ nix_tests = \
   recursive.sh \
   flakes.sh
   # parallel.sh
+  # build-remote-content-addressed-fixed.sh \
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 

From 069340179e91202b2d3adc5ea1e3023ba2a51691 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 18 Aug 2020 15:15:35 +0200
Subject: [PATCH 112/882] Improve nix.1 manpage generator

---
 doc/manual/generate-manpage.jq | 72 +++++++++++++++++-----------------
 src/nix/main.cc                |  5 +++
 2 files changed, 40 insertions(+), 37 deletions(-)

diff --git a/doc/manual/generate-manpage.jq b/doc/manual/generate-manpage.jq
index cf06da0c8..d3cf1c601 100644
--- a/doc/manual/generate-manpage.jq
+++ b/doc/manual/generate-manpage.jq
@@ -1,42 +1,40 @@
 def show_flags:
-
-.flags
-| map_values(select(.category != "config"))
-| to_entries
-| map(
-    "  - `--" + .key + "`"
-    + (if .value.shortName then " / `" + .value.shortName + "`" else "" end)
-    + (if .value.labels then " " + (.value.labels | map("*" + . + "*") | join(" ")) else "" end)
-    + "  \n"
-    + "    " + .value.description + "\n\n")
-| join("")
-;
+  .flags
+  | map_values(select(.category != "config"))
+  | to_entries
+  | map(
+      "  - `--" + .key + "`"
+      + (if .value.shortName then " / `" + .value.shortName + "`" else "" end)
+      + (if .value.labels then " " + (.value.labels | map("*" + . + "*") | join(" ")) else "" end)
+      + "  \n"
+      + "    " + .value.description + "\n\n")
+  | join("")
+  ;
 
 def show_synopsis:
+  "`" + .command + "` " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n\n"
+  ;
 
-"`" + .command + "` " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n"
+def show_command:
+  . as $top |
+  .section + " Name\n\n"
+  + "`" + .command + "` - " + .def.description + "\n\n"
+  + .section + " Synopsis\n\n"
+  + ({"command": .command, "args": .def.args} | show_synopsis)
+  + (if (.def.flags | length) > 0 then
+      .section + " Flags\n\n"
+      + (.def | show_flags)
+    else "" end)
+  + (if (.def.examples | length) > 0 then
+      .section + " Examples\n\n"
+      + (.def.examples | map(.description + "\n\n```console\n" + .command + "\n```\n" ) | join("\n"))
+      + "\n"
+     else "" end)
+  + (if .def.commands then .def.commands | to_entries | map(
+      "# Subcommand `" + ($top.command + " " + .key) + "`\n\n"
+      + ({"command": ($top.command + " " + .key), "section": "##", "def": .value} | show_command)
+    ) | join("") else "" end)
+  ;
 
-;
-
-"# Synopsis\n\n"
-+ ({"command": "nix", "args": .args} | show_synopsis)
-+ "\n"
-+ "# Common flags\n\n"
-+ show_flags
-+ (.commands | to_entries | map(
-    "# Operation `" + .key + "`\n\n"
-    + "## Synopsis\n\n"
-    + ({"command": ("nix " + .key), "args": .value.args} | show_synopsis)
-    + "\n"
-    + "## Description\n\n"
-    + .value.description + "\n\n"
-    + (if (.value.flags | length) > 0 then
-        "## Flags\n\n"
-        + (.value | show_flags)
-      else "" end)
-    + (if (.value.examples | length) > 0 then
-        "## Examples\n\n"
-        + (.value.examples | map("- " + .description + "\n\n  ```console\n  " + .command + "\n  ```\n" ) | join("\n"))
-        + "\n"
-       else "" end)
-  ) | join(""))
+"Title: nix\n\n"
++ ({"command": "nix", "section": "#", "def": .} | show_command)
diff --git a/src/nix/main.cc b/src/nix/main.cc
index fa04bebea..0c4fd46fd 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -142,6 +142,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         printHelp(programName, std::cout);
         throw Exit();
     }
+
+    std::string description() override
+    {
+        return "a tool for reproducible and declarative configuration management";
+    }
 };
 
 void mainWrapped(int argc, char * * argv)

From 8a97b11374c5165d3602648c5c0327f103072105 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Aug 2020 12:31:18 +0200
Subject: [PATCH 113/882] Improve margins between sections

The default CSS puts almost no space between sections, but a lot of
space between subsections. This flips that around.
---
 doc/manual/book.toml  | 2 ++
 doc/manual/custom.css | 7 +++++++
 doc/manual/local.mk   | 2 +-
 3 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 doc/manual/book.toml
 create mode 100644 doc/manual/custom.css

diff --git a/doc/manual/book.toml b/doc/manual/book.toml
new file mode 100644
index 000000000..fee41dfb3
--- /dev/null
+++ b/doc/manual/book.toml
@@ -0,0 +1,2 @@
+[output.html]
+additional-css = ["custom.css"]
diff --git a/doc/manual/custom.css b/doc/manual/custom.css
new file mode 100644
index 000000000..69d48d4a7
--- /dev/null
+++ b/doc/manual/custom.css
@@ -0,0 +1,7 @@
+h1:not(:first-of-type) {
+    margin-top: 1.3em;
+}
+
+h2 {
+    margin-top: 1em;
+}
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 8f917316d..04c57b23b 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -33,7 +33,7 @@ $(d)/nix.json: $(bindir)/nix
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS)
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css
 	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
 	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
 

From 34b22e012350186925e513f34b1292858a81c932 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Aug 2020 14:21:27 +0200
Subject: [PATCH 114/882] Change option descriptions to Markdown

---
 src/libexpr/common-eval-args.cc | 2 +-
 src/libmain/common-args.cc      | 6 +++---
 src/nix/installables.cc         | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 6b48ead1f..10c1a6975 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -29,7 +29,7 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "include",
         .shortName = 'I',
-        .description = "add a path to the list of locations used to look up <...> file names",
+        .description = "add a path to the list of locations used to look up `<...>` file names",
         .labels = {"path"},
         .handler = {[&](std::string s) { searchPath.push_back(s); }}
     });
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 09f4cd133..3411e2d7a 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -28,7 +28,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "option",
-        .description = "set a Nix configuration option (overriding nix.conf)",
+        .description = "set a Nix configuration option (overriding `nix.conf`)",
         .labels = {"name", "value"},
         .handler = {[](std::string name, std::string value) {
             try {
@@ -51,8 +51,8 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "log-format",
-        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
-                        "or \"bar-with-logs\"",
+        .description = "format of log output; `raw`, `internal-json`, `bar` "
+                        "or `bar-with-logs`",
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index d34f87982..1f1ed680f 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -76,7 +76,7 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "override a specific flake input (e.g. 'dwarffs/nixpkgs')",
+        .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)",
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
             lockFlags.inputOverrides.insert_or_assign(
@@ -116,7 +116,7 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "file",
         .shortName = 'f',
-        .description = "evaluate FILE rather than the default",
+        .description = "evaluate *file* rather than the default",
         .labels = {"file"},
         .handler = {&file},
         .completer = completePath
@@ -124,7 +124,7 @@ SourceExprCommand::SourceExprCommand()
 
     addFlag({
         .longName ="expr",
-        .description = "evaluate attributes from EXPR",
+        .description = "evaluate attributes from *expr*",
         .labels = {"expr"},
         .handler = {&expr}
     });

From c8fa39324ad7a56a78f8c6f55c42f8f49dbbbf9a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Aug 2020 18:28:04 +0200
Subject: [PATCH 115/882] Generate the nix.conf docs from the source code

This means we don't have two (divergent) sets of option descriptions
anymore.
---
 .gitignore                                    |   2 +
 doc/manual/generate-options.jq                |  11 +
 doc/manual/local.mk                           |   7 +
 .../src/command-ref/conf-file-prefix.md       |  39 +
 doc/manual/src/command-ref/conf-file.md       | 691 ----------------
 src/libexpr/eval.hh                           |  51 +-
 src/libstore/filetransfer.hh                  |  27 +-
 src/libstore/globals.hh                       | 759 +++++++++++++++---
 src/libutil/config.cc                         |  39 +-
 src/libutil/logging.hh                        |  10 +-
 10 files changed, 802 insertions(+), 834 deletions(-)
 create mode 100644 doc/manual/generate-options.jq
 create mode 100644 doc/manual/src/command-ref/conf-file-prefix.md
 delete mode 100644 doc/manual/src/command-ref/conf-file.md

diff --git a/.gitignore b/.gitignore
index b8028e665..44dbaa5d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,7 +26,9 @@ perl/Makefile.config
 /doc/manual/*.5
 /doc/manual/*.8
 /doc/manual/nix.json
+/doc/manual/conf-file.json
 /doc/manual/src/command-ref/nix.md
+/doc/manual/src/command-ref/conf-file.md
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/generate-options.jq b/doc/manual/generate-options.jq
new file mode 100644
index 000000000..3ee51ddea
--- /dev/null
+++ b/doc/manual/generate-options.jq
@@ -0,0 +1,11 @@
+. | to_entries | sort_by(.key) | map(
+  "  - `" + .key + "`  \n"
+  + (.value.description | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
+  + "    **Default**: " + (
+      if .value.value == "" or .value.value == []
+      then "*empty*"
+      elif (.value.value | type) == "array"
+      then "`" + (.value.value | join(" ")) + "`"
+      else "`" + (.value.value | tostring) + "`" end)
+  + "\n\n"
+) | join("")
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 04c57b23b..dcc02d538 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -27,9 +27,16 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 $(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.jq
 	jq -r -f doc/manual/generate-manpage.jq $< > $@
 
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.jq $(d)/src/command-ref/conf-file-prefix.md
+	cat doc/manual/src/command-ref/conf-file-prefix.md > $@
+	jq -r -f doc/manual/generate-options.jq $< >> $@
+
 $(d)/nix.json: $(bindir)/nix
 	$(trace-gen) $(bindir)/nix dump-args > $@
 
+$(d)/conf-file.json: $(bindir)/nix
+	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@
+
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
new file mode 100644
index 000000000..04c6cd859
--- /dev/null
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -0,0 +1,39 @@
+Title: nix.conf
+
+# Name
+
+`nix.conf` - Nix configuration file
+
+# Description
+
+By default Nix reads settings from the following places:
+
+  - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e.
+    `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if
+    `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded
+    to the Nix daemon. The client assumes that the daemon has already
+    loaded them.
+
+  - If `NIX_USER_CONF_FILES` is set, then each path separated by `:`
+    will be loaded in reverse order.
+
+    Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
+    and `XDG_CONFIG_HOME`. If these are unset, it will look in
+    `$HOME/.config/nix.conf`.
+
+The configuration files consist of `name =
+value` pairs, one per line. Other files can be included with a line like
+`include
+path`, where *path* is interpreted relative to the current conf file and
+a missing file is an error unless `!include` is used instead. Comments
+start with a `#` character. Here is an example configuration file:
+
+    keep-outputs = true       # Nice for developers
+    keep-derivations = true   # Idem
+
+You can override settings on the command line using the `--option` flag,
+e.g. `--option keep-outputs
+false`.
+
+The following settings are currently available:
+
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
deleted file mode 100644
index fdc8f5265..000000000
--- a/doc/manual/src/command-ref/conf-file.md
+++ /dev/null
@@ -1,691 +0,0 @@
-Title: nix.conf
-
-# Name
-
-`nix.conf` - Nix configuration file
-
-# Description
-
-By default Nix reads settings from the following places:
-
-  - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e.
-    `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if
-    `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded
-    to the Nix daemon. The client assumes that the daemon has already
-    loaded them.
-
-  - If `NIX_USER_CONF_FILES` is set, then each path separated by `:`
-    will be loaded in reverse order.
-
-    Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
-    and `XDG_CONFIG_HOME`. If these are unset, it will look in
-    `$HOME/.config/nix.conf`.
-
-The configuration files consist of `name =
-value` pairs, one per line. Other files can be included with a line like
-`include
-path`, where *path* is interpreted relative to the current conf file and
-a missing file is an error unless `!include` is used instead. Comments
-start with a `#` character. Here is an example configuration file:
-
-    keep-outputs = true       # Nice for developers
-    keep-derivations = true   # Idem
-
-You can override settings on the command line using the `--option` flag,
-e.g. `--option keep-outputs
-false`.
-
-The following settings are currently available:
-
-  - `allowed-uris`  
-    A list of URI prefixes to which access is allowed in restricted
-    evaluation mode. For example, when set to
-    `https://github.com/NixOS`, builtin functions such as `fetchGit` are
-    allowed to access `https://github.com/NixOS/patchelf.git`.
-
-  - `allow-import-from-derivation`  
-    By default, Nix allows you to `import` from a derivation, allowing
-    building at evaluation time. With this option set to false, Nix will
-    throw an error when evaluating an expression that uses this feature,
-    allowing users to ensure their evaluation will not require any
-    builds to take place.
-
-  - `allow-new-privileges`  
-    (Linux-specific.) By default, builders on Linux cannot acquire new
-    privileges by calling setuid/setgid programs or programs that have
-    file capabilities. For example, programs such as `sudo` or `ping`
-    will fail. (Note that in sandbox builds, no such programs are
-    available unless you bind-mount them into the sandbox via the
-    `sandbox-paths` option.) You can allow the use of such programs by
-    enabling this option. This is impure and usually undesirable, but
-    may be useful in certain scenarios (e.g. to spin up containers or
-    set up userspace network interfaces in tests).
-
-  - `allowed-users`  
-    A list of names of users (separated by whitespace) that are allowed
-    to connect to the Nix daemon. As with the `trusted-users` option,
-    you can specify groups by prefixing them with `@`. Also, you can
-    allow all users by specifying `*`. The default is `*`.
-    
-    Note that trusted users are always allowed to connect.
-
-  - `auto-optimise-store`  
-    If set to `true`, Nix automatically detects files in the store
-    that have identical contents, and replaces them with hard links to
-    a single copy. This saves disk space. If set to `false` (the
-    default), you can still run `nix-store --optimise` to get rid of
-    duplicate files.
-
-  - `builders`  
-    A list of machines on which to perform builds.
-
-  - `builders-use-substitutes`  
-    If set to `true`, Nix will instruct remote build machines to use
-    their own binary substitutes if available. In practical terms, this
-    means that remote hosts will fetch as many build dependencies as
-    possible from their own substitutes (e.g, from `cache.nixos.org`),
-    instead of waiting for this host to upload them all. This can
-    drastically reduce build times if the network connection between
-    this computer and the remote build host is slow. Defaults to
-    `false`.
-
-  - `build-users-group`  
-    This options specifies the Unix group containing the Nix build user
-    accounts. In multi-user Nix installations, builds should not be
-    performed by the Nix account since that would allow users to
-    arbitrarily modify the Nix store and database by supplying specially
-    crafted builders; and they cannot be performed by the calling user
-    since that would allow him/her to influence the build result.
-    
-    Therefore, if this option is non-empty and specifies a valid group,
-    builds will be performed under the user accounts that are a member
-    of the group specified here (as listed in `/etc/group`). Those user
-    accounts should not be used for any other purpose\!
-    
-    Nix will never run two builds under the same user account at the
-    same time. This is to prevent an obvious security hole: a malicious
-    user writing a Nix expression that modifies the build result of a
-    legitimate Nix expression being built by another user. Therefore it
-    is good to have as many Nix build user accounts as you can spare.
-    (Remember: uids are cheap.)
-    
-    The build users should have permission to create files in the Nix
-    store, but not delete them. Therefore, `/nix/store` should be owned
-    by the Nix account, its group should be the group specified here,
-    and its mode should be `1775`.
-    
-    If the build users group is empty, builds will be performed under
-    the uid of the Nix process (that is, the uid of the caller if
-    `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
-    `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
-    multi-user settings with untrusted users.
-
-  - `compress-build-log`  
-    If set to `true` (the default), build logs written to
-    `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2.
-    Otherwise, they will not be compressed.
-
-  - `connect-timeout`  
-    The timeout (in seconds) for establishing connections in the binary
-    cache substituter. It corresponds to `curl`’s `--connect-timeout`
-    option.
-
-  - `cores`  
-    Sets the value of the `NIX_BUILD_CORES` environment variable in the
-    invocation of builders. Builders can use this variable at their
-    discretion to control the maximum amount of parallelism. For
-    instance, in Nixpkgs, if the derivation attribute
-    `enableParallelBuilding` is set to `true`, the builder passes the
-    `-jN` flag to GNU Make. It can be overridden using the `--cores`
-    command line switch and defaults to `1`. The value `0` means that
-    the builder should use all available CPU cores in the system.
-
-  - `diff-hook`  
-    Absolute path to an executable capable of diffing build
-    results. The hook is executed if `run-diff-hook` is true, and the
-    output of a build is known to not be the same. This program is not
-    executed to determine if two results are the same.
-    
-    The diff hook is executed by the same user and group who ran the
-    build. However, the diff hook does not have write access to the
-    store path just built.
-    
-    The diff hook program receives three parameters:
-    
-    1.  A path to the previous build's results
-    
-    2.  A path to the current build's results
-    
-    3.  The path to the build's derivation
-    
-    4.  The path to the build's scratch directory. This directory will
-        exist only if the build was run with `--keep-failed`.
-    
-    The stderr and stdout output from the diff hook will not be
-    displayed to the user. Instead, it will print to the nix-daemon's
-    log.
-    
-    When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
-    configuration file, and cannot be passed at the command line.
-
-  - `enforce-determinism`  
-    See `repeat`.
-
-  - `extra-sandbox-paths`  
-    A list of additional paths appended to `sandbox-paths`. Useful if
-    you want to extend its default value.
-
-  - `extra-platforms`  
-    Platforms other than the native one which this machine is capable of
-    building for. This can be useful for supporting additional
-    architectures on compatible machines: i686-linux can be built on
-    x86\_64-linux machines (and the default for this setting reflects
-    this); armv7 is backwards-compatible with armv6 and armv5tel; some
-    aarch64 machines can also natively run 32-bit ARM code; and
-    qemu-user may be used to support non-native platforms (though this
-    may be slow and buggy). Most values for this are not enabled by
-    default because build systems will often misdetect the target
-    platform and generate incompatible code, so you may wish to
-    cross-check the results of using this option against proper
-    natively-built versions of your derivations.
-
-  - `extra-substituters`  
-    Additional binary caches appended to those specified in
-    `substituters`. When used by unprivileged users, untrusted
-    substituters (i.e. those not listed in `trusted-substituters`) are
-    silently ignored.
-
-  - `fallback`  
-    If set to `true`, Nix will fall back to building from source if a
-    binary substitute fails. This is equivalent to the `--fallback`
-    flag. The default is `false`.
-
-  - `fsync-metadata`  
-    If set to `true`, changes to the Nix store metadata (in
-    `/nix/var/nix/db`) are synchronously flushed to disk. This improves
-    robustness in case of system crashes, but reduces performance. The
-    default is `true`.
-
-  - `hashed-mirrors`  
-    A list of web servers used by `builtins.fetchurl` to obtain files by
-    hash. The default is `http://tarballs.nixos.org/`. Given a hash type
-    *ht* and a base-16 hash *h*, Nix will try to download the file from
-    *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
-    they have disappeared from their original URI. For example, given
-    the default mirror `http://tarballs.nixos.org/`, when building the
-    derivation
-
-    ```nix
-    builtins.fetchurl {
-      url = "https://example.org/foo-1.2.3.tar.xz";
-      sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-    }
-    ```
-    
-    Nix will attempt to download this file from
-    `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
-    first. If it is not available there, if will try the original URI.
-
-  - `http-connections`  
-    The maximum number of parallel TCP connections used to fetch files
-    from binary caches and by other downloads. It defaults to 25. 0
-    means no limit.
-
-  - `keep-build-log`  
-    If set to `true` (the default), Nix will write the build log of a
-    derivation (i.e. the standard output and error of its builder) to
-    the directory `/nix/var/log/nix/drvs`. The build log can be
-    retrieved using the command `nix-store -l path`.
-
-  - `keep-derivations`  
-    If `true` (default), the garbage collector will keep the derivations
-    from which non-garbage store paths were built. If `false`, they will
-    be deleted unless explicitly registered as a root (or reachable from
-    other roots).
-    
-    Keeping derivation around is useful for querying and traceability
-    (e.g., it allows you to ask with what dependencies or options a
-    store path was built), so by default this option is on. Turn it off
-    to save a bit of disk space (or a lot if `keep-outputs` is also
-    turned on).
-
-  - `keep-env-derivations`  
-    If `false` (default), derivations are not stored in Nix user
-    environments. That is, the derivations of any build-time-only
-    dependencies may be garbage-collected.
-    
-    If `true`, when you add a Nix derivation to a user environment, the
-    path of the derivation is stored in the user environment. Thus, the
-    derivation will not be garbage-collected until the user environment
-    generation is deleted (`nix-env --delete-generations`). To prevent
-    build-time-only dependencies from being collected, you should also
-    turn on `keep-outputs`.
-    
-    The difference between this option and `keep-derivations` is that
-    this one is “sticky”: it applies to any user environment created
-    while this option was enabled, while `keep-derivations` only applies
-    at the moment the garbage collector is run.
-
-  - `keep-outputs`  
-    If `true`, the garbage collector will keep the outputs of
-    non-garbage derivations. If `false` (default), outputs will be
-    deleted unless they are GC roots themselves (or reachable from other
-    roots).
-    
-    In general, outputs must be registered as roots separately. However,
-    even if the output of a derivation is registered as a root, the
-    collector will still delete store paths that are used only at build
-    time (e.g., the C compiler, or source tarballs downloaded from the
-    network). To prevent it from doing so, set this option to `true`.
-
-  - `max-build-log-size`  
-    This option defines the maximum number of bytes that a builder can
-    write to its stdout/stderr. If the builder exceeds this limit, it’s
-    killed. A value of `0` (the default) means that there is no limit.
-
-  - `max-free`  
-    When a garbage collection is triggered by the `min-free` option, it
-    stops as soon as `max-free` bytes are available. The default is
-    infinity (i.e. delete all garbage).
-
-  - `max-jobs`  
-    This option defines the maximum number of jobs that Nix will try to
-    build in parallel. The default is `1`. The special value `auto`
-    causes Nix to use the number of CPUs in your system. `0` is useful
-    when using remote builders to prevent any local builds (except for
-    `preferLocalBuild` derivation attribute which executes locally
-    regardless). It can be overridden using the `--max-jobs` (`-j`)
-    command line switch.
-
-  - `max-silent-time`  
-    This option defines the maximum number of seconds that a builder can
-    go without producing any data on standard output or standard error.
-    This is useful (for instance in an automated build system) to catch
-    builds that are stuck in an infinite loop, or to catch remote builds
-    that are hanging due to network problems. It can be overridden using
-    the `--max-silent-time` command line switch.
-    
-    The value `0` means that there is no timeout. This is also the
-    default.
-
-  - `min-free`  
-    When free disk space in `/nix/store` drops below `min-free` during a
-    build, Nix performs a garbage-collection until `max-free` bytes are
-    available or there is no more garbage. A value of `0` (the default)
-    disables this feature.
-
-  - `narinfo-cache-negative-ttl`  
-    The TTL in seconds for negative lookups. If a store path is queried
-    from a substituter but was not found, there will be a negative
-    lookup cached in the local disk cache database for the specified
-    duration.
-
-  - `narinfo-cache-positive-ttl`  
-    The TTL in seconds for positive lookups. If a store path is queried
-    from a substituter, the result of the query will be cached in the
-    local disk cache database including some of the NAR metadata. The
-    default TTL is a month, setting a shorter TTL for positive lookups
-    can be useful for binary caches that have frequent garbage
-    collection, in which case having a more frequent cache invalidation
-    would prevent trying to pull the path again and failing with a hash
-    mismatch if the build isn't reproducible.
-
-  - `netrc-file`  
-    If set to an absolute path to a `netrc` file, Nix will use the HTTP
-    authentication credentials in this file when trying to download from
-    a remote host through HTTP or HTTPS. Defaults to
-    `$NIX_CONF_DIR/netrc`.
-    
-    The `netrc` file consists of a list of accounts in the following
-    format:
-    
-        machine my-machine
-        login my-username
-        password my-password
-    
-    For the exact syntax, see [the `curl`
-    documentation](https://ec.haxx.se/usingcurl-netrc.html).
-    
-    > **Note**
-    > 
-    > This must be an absolute path, and `~` is not resolved. For
-    > example, `~/.netrc` won't resolve to your home directory's
-    > `.netrc`.
-
-  - `plugin-files`  
-    A list of plugin files to be loaded by Nix. Each of these files will
-    be dlopened by Nix, allowing them to affect execution through static
-    initialization. In particular, these plugins may construct static
-    instances of RegisterPrimOp to add new primops or constants to the
-    expression language, RegisterStoreImplementation to add new store
-    implementations, RegisterCommand to add new subcommands to the `nix`
-    command, and RegisterSetting to add new nix config settings. See the
-    constructors for those types for more details.
-    
-    Since these files are loaded into the same address space as Nix
-    itself, they must be DSOs compatible with the instance of Nix
-    running at the time (i.e. compiled against the same headers, not
-    linked to any incompatible libraries). They should not be linked to
-    any Nix libs directly, as those will be available already at load
-    time.
-    
-    If an entry in the list is a directory, all files in the directory
-    are loaded as plugins (non-recursively).
-
-  - `pre-build-hook`  
-    If set, the path to a program that can set extra derivation-specific
-    settings for this system. This is used for settings that can't be
-    captured by the derivation model itself and are too variable between
-    different versions of the same system to be hard-coded into nix.
-    
-    The hook is passed the derivation path and, if sandboxes are
-    enabled, the sandbox directory. It can then modify the sandbox and
-    send a series of commands to modify various settings to stdout. The
-    currently recognized commands are:
-    
-      - `extra-sandbox-paths`  
-        Pass a list of files and directories to be included in the
-        sandbox for this build. One entry per line, terminated by an
-        empty line. Entries have the same format as `sandbox-paths`.
-
-  - `post-build-hook`  
-    Optional. The path to a program to execute after each build.
-    
-    This option is only settable in the global `nix.conf`, or on the
-    command line by trusted users.
-    
-    When using the nix-daemon, the daemon executes the hook as `root`.
-    If the nix-daemon is not involved, the hook runs as the user
-    executing the nix-build.
-    
-      - The hook executes after an evaluation-time build.
-    
-      - The hook does not execute on substituted paths.
-    
-      - The hook's output always goes to the user's terminal.
-    
-      - If the hook fails, the build succeeds but no further builds
-        execute.
-    
-      - The hook executes synchronously, and blocks other builds from
-        progressing while it runs.
-    
-    The program executes with no arguments. The program's environment
-    contains the following environment variables:
-    
-      - `DRV_PATH`  
-        The derivation for the built paths.
-        
-        Example:
-        `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
-    
-      - `OUT_PATHS`  
-        Output paths of the built derivation, separated by a space
-        character.
-        
-        Example:
-        `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
-        /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
-        /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
-        /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
-        /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
-
-  - `repeat`  
-    How many times to repeat builds to check whether they are
-    deterministic. The default value is 0. If the value is non-zero,
-    every build is repeated the specified number of times. If the
-    contents of any of the runs differs from the previous ones and
-    `enforce-determinism` is true, the build is rejected and the
-    resulting store paths are not registered as “valid” in Nix’s
-    database.
-
-  - `require-sigs`  
-    If set to `true` (the default), any non-content-addressed path added
-    or copied to the Nix store (e.g. when substituting from a binary
-    cache) must have a valid signature, that is, be signed using one of
-    the keys listed in `trusted-public-keys` or `secret-key-files`. Set
-    to `false` to disable signature checking.
-
-  - `restrict-eval`  
-    If set to `true`, the Nix evaluator will not allow access to any
-    files outside of the Nix search path (as set via the `NIX_PATH`
-    environment variable or the `-I` option), or to URIs outside of
-    `allowed-uri`. The default is `false`.
-
-  - `run-diff-hook`  
-    If true, enable the execution of the `diff-hook` program.
-    
-    When using the Nix daemon, `run-diff-hook` must be set in the
-    `nix.conf` configuration file, and cannot be passed at the command
-    line.
-
-  - `sandbox`  
-    If set to `true`, builds will be performed in a *sandboxed
-    environment*, i.e., they’re isolated from the normal file system
-    hierarchy and will only see their dependencies in the Nix store,
-    the temporary build directory, private versions of `/proc`,
-    `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths
-    configured with the `sandbox-paths` option. This is useful to
-    prevent undeclared dependencies on files in directories such as
-    `/usr/bin`. In addition, on Linux, builds run in private PID,
-    mount, network, IPC and UTS namespaces to isolate them from other
-    processes in the system (except that fixed-output derivations do
-    not run in private network namespace to ensure they can access the
-    network).
-    
-    Currently, sandboxing only work on Linux and macOS. The use of a
-    sandbox requires that Nix is run as root (so you should use the
-    “build users” feature to perform the actual builds under different
-    users than root).
-    
-    If this option is set to `relaxed`, then fixed-output derivations
-    and derivations that have the `__noChroot` attribute set to `true`
-    do not run in sandboxes.
-    
-    The default is `true` on Linux and `false` on all other platforms.
-
-  - `sandbox-dev-shm-size`  
-    This option determines the maximum size of the `tmpfs` filesystem
-    mounted on `/dev/shm` in Linux sandboxes. For the format, see the
-    description of the `size` option of `tmpfs` in mount8. The default
-    is `50%`.
-
-  - `sandbox-paths`  
-    A list of paths bind-mounted into Nix sandbox environments. You can
-    use the syntax `target=source` to mount a path in a different
-    location in the sandbox; for instance, `/bin=/nix-bin` will mount
-    the path `/nix-bin` as `/bin` inside the sandbox. If *source* is
-    followed by `?`, then it is not an error if *source* does not exist;
-    for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
-    only be mounted in the sandbox if it exists in the host filesystem.
-    
-    Depending on how Nix was built, the default value for this option
-    may be empty or provide `/bin/sh` as a bind-mount of `bash`.
-
-  - `secret-key-files`  
-    A whitespace-separated list of files containing secret (private)
-    keys. These are used to sign locally-built paths. They can be
-    generated using `nix-store --generate-binary-cache-key`. The
-    corresponding public key can be distributed to other users, who
-    can add it to `trusted-public-keys` in their `nix.conf`.
-
-  - `show-trace`  
-    Causes Nix to print out a stack trace in case of Nix expression
-    evaluation errors.
-
-  - `substitute`  
-    If set to `true` (default), Nix will use binary substitutes if
-    available. This option can be disabled to force building from
-    source.
-
-  - `stalled-download-timeout`  
-    The timeout (in seconds) for receiving data from servers during
-    download. Nix cancels idle downloads after this timeout's duration.
-
-  - `substituters`  
-    A list of URLs of substituters, separated by whitespace. The default
-    is `https://cache.nixos.org`.
-
-  - `system`  
-    This option specifies the canonical Nix system name of the current
-    installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
-    build derivations whose `system` attribute equals the value
-    specified here. In general, it never makes sense to modify this
-    value from its default, since you can use it to ‘lie’ about the
-    platform you are building on (e.g., perform a Mac OS build on a
-    Linux machine; the result would obviously be wrong). It only makes
-    sense if the Nix binaries can run on multiple platforms, e.g.,
-    ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
-    
-    It defaults to the canonical Nix system name detected by `configure`
-    at build time.
-
-  - `system-features`  
-    A set of system “features” supported by this machine, e.g. `kvm`.
-    Derivations can express a dependency on such features through the
-    derivation attribute `requiredSystemFeatures`. For example, the
-    attribute
-    
-        requiredSystemFeatures = [ "kvm" ];
-    
-    ensures that the derivation can only be built on a machine with the
-    `kvm` feature.
-    
-    This setting by default includes `kvm` if `/dev/kvm` is accessible,
-    and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
-    that are used in Nixpkgs to route builds to specific machines.
-
-  - `tarball-ttl`  
-    Default: `3600` seconds.
-    
-    The number of seconds a downloaded tarball is considered fresh. If
-    the cached tarball is stale, Nix will check whether it is still up
-    to date using the ETag header. Nix will download a new version if
-    the ETag header is unsupported, or the cached ETag doesn't match.
-    
-    Setting the TTL to `0` forces Nix to always check if the tarball is
-    up to date.
-    
-    Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
-    
-    Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
-    `fetchTarball`, and `fetchurl` respect this TTL.
-
-  - `timeout`  
-    This option defines the maximum number of seconds that a builder can
-    run. This is useful (for instance in an automated build system) to
-    catch builds that are stuck in an infinite loop but keep writing to
-    their standard output or standard error. It can be overridden using
-    the `--timeout` command line switch.
-    
-    The value `0` means that there is no timeout. This is also the
-    default.
-
-  - `trace-function-calls`  
-    Default: `false`.
-    
-    If set to `true`, the Nix evaluator will trace every function call.
-    Nix will print a log message at the "vomit" level for every function
-    entrance and function exit.
-    
-        function-trace entered undefined position at 1565795816999559622
-        function-trace exited undefined position at 1565795816999581277
-        function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
-        function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
-    
-    The `undefined position` means the function call is a builtin.
-    
-    Use the `contrib/stack-collapse.py` script distributed with the Nix
-    source code to convert the trace logs in to a format suitable for
-    `flamegraph.pl`.
-
-  - `trusted-public-keys`  
-    A whitespace-separated list of public keys. When paths are copied
-    from another Nix store (such as a binary cache), they must be
-    signed with one of these keys. For example:
-    `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
-
-  - `trusted-substituters`  
-    A list of URLs of substituters, separated by whitespace. These are
-    not used by default, but can be enabled by users of the Nix daemon
-    by specifying `--option substituters urls` on the command
-    line. Unprivileged users are only allowed to pass a subset of the
-    URLs listed in `substituters` and `trusted-substituters`.
-
-  - `trusted-users`  
-    A list of names of users (separated by whitespace) that have
-    additional rights when connecting to the Nix daemon, such as the
-    ability to specify additional binary caches, or to import unsigned
-    NARs. You can also specify groups by prefixing them with `@`; for
-    instance, `@wheel` means all users in the `wheel` group. The default
-    is `root`.
-    
-    > **Warning**
-    > 
-    > Adding a user to `trusted-users` is essentially equivalent to
-    > giving that user root access to the system. For example, the user
-    > can set `sandbox-paths` and thereby obtain read access to
-    > directories that are otherwise inacessible to them.
-
-## Deprecated Settings
-
-  - `binary-caches`  
-    *Deprecated:* `binary-caches` is now an alias to `substituters`.
-
-  - `binary-cache-public-keys`  
-    *Deprecated:* `binary-cache-public-keys` is now an alias `trusted-public-keys`.
-
-  - `build-compress-log`  
-    *Deprecated:* `build-compress-log` is now an alias to `compress-build-log`.
-
-  - `build-cores`  
-    *Deprecated:* `build-cores` is now an alias to `cores`.
-
-  - `build-extra-chroot-dirs`  
-    *Deprecated:* `build-extra-chroot-dirs` is now an alias to `extra-sandbox-paths`.
-
-  - `build-extra-sandbox-paths`  
-    *Deprecated:* `build-extra-sandbox-paths` is now an alias to `extra-sandbox-paths`.
-
-  - `build-fallback`  
-    *Deprecated:* `build-fallback` is now an alias to `fallback`.
-
-  - `build-max-jobs`  
-    *Deprecated:* `build-max-jobs` is now an alias to `max-jobs`.
-
-  - `build-max-log-size`  
-    *Deprecated:* `build-max-log-size` is now an alias to `max-build-log-size`.
-
-  - `build-max-silent-time`  
-    *Deprecated:* `build-max-silent-time` is now an alias to `max-silent-time`.
-
-  - `build-repeat`  
-    *Deprecated:* `build-repeat` is now an alias to `repeat`.
-
-  - `build-timeout`  
-    *Deprecated:* `build-timeout` is now an alias to `timeout`.
-
-  - `build-use-chroot`  
-    *Deprecated:* `build-use-chroot` is now an alias to `sandbox`.
-
-  - `build-use-sandbox`  
-    *Deprecated:* `build-use-sandbox` is now an alias to `sandbox`.
-
-  - `build-use-substitutes`  
-    *Deprecated:* `build-use-substitutes` is now an alias to `substitute`.
-
-  - `gc-keep-derivations`  
-    *Deprecated:* `gc-keep-derivations` is now an alias to `keep-derivations`.
-
-  - `gc-keep-outputs`  
-    *Deprecated:* `gc-keep-outputs` is now an alias to `keep-outputs`.
-
-  - `env-keep-derivations`  
-    *Deprecated:* `env-keep-derivations` is now an alias to `keep-env-derivations`.
-
-  - `extra-binary-caches`  
-    *Deprecated:* `extra-binary-caches` is now an alias to `extra-substituters`.
-
-  - `trusted-binary-caches`  
-    *Deprecated:* `trusted-binary-caches` is now an alias to `trusted-substituters`.
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 5855b4ef2..1db3aa766 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -357,24 +357,57 @@ struct EvalSettings : Config
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
-    Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
-        "List of directories to be searched for <...> file references."};
+    Setting<Strings> nixPath{
+        this, getDefaultNixPath(), "nix-path",
+        "List of directories to be searched for `<...>` file references."};
 
-    Setting<bool> restrictEval{this, false, "restrict-eval",
-        "Whether to restrict file system access to paths in $NIX_PATH, "
-        "and network access to the URI prefixes listed in 'allowed-uris'."};
+    Setting<bool> restrictEval{
+        this, false, "restrict-eval",
+        R"(
+          If set to `true`, the Nix evaluator will not allow access to any
+          files outside of the Nix search path (as set via the `NIX_PATH`
+          environment variable or the `-I` option), or to URIs outside of
+          `allowed-uri`. The default is `false`.
+        )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",
         "Whether to restrict file system and network access to files specified by cryptographic hash."};
 
-    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
-        "Whether the evaluator allows importing the result of a derivation."};
+    Setting<bool> enableImportFromDerivation{
+        this, true, "allow-import-from-derivation",
+        R"(
+          By default, Nix allows you to `import` from a derivation, allowing
+          building at evaluation time. With this option set to false, Nix will
+          throw an error when evaluating an expression that uses this feature,
+          allowing users to ensure their evaluation will not require any
+          builds to take place.
+        )"};
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
-        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+        R"(
+          A list of URI prefixes to which access is allowed in restricted
+          evaluation mode. For example, when set to
+          `https://github.com/NixOS`, builtin functions such as `fetchGit` are
+          allowed to access `https://github.com/NixOS/patchelf.git`.
+        )"};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+        R"(
+          If set to `true`, the Nix evaluator will trace every function call.
+          Nix will print a log message at the "vomit" level for every function
+          entrance and function exit.
+
+              function-trace entered undefined position at 1565795816999559622
+              function-trace exited undefined position at 1565795816999581277
+              function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+              function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+
+          The `undefined position` means the function call is a builtin.
+
+          Use the `contrib/stack-collapse.py` script distributed with the Nix
+          source code to convert the trace logs in to a format suitable for
+          `flamegraph.pl`.
+        )"};
 
     Setting<bool> useEvalCache{this, true, "eval-cache",
         "Whether to use the flake evaluation cache."};
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index 25ade0add..0d608c8d8 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -17,15 +17,30 @@ struct FileTransferSettings : Config
     Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
         "String appended to the user agent in HTTP requests."};
 
-    Setting<size_t> httpConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
+    Setting<size_t> httpConnections{
+        this, 25, "http-connections",
+        R"(
+          The maximum number of parallel TCP connections used to fetch
+          files from binary caches and by other downloads. It defaults
+          to 25. 0 means no limit.
+        )",
         {"binary-caches-parallel-connections"}};
 
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+    Setting<unsigned long> connectTimeout{
+        this, 0, "connect-timeout",
+        R"(
+          The timeout (in seconds) for establishing connections in the
+          binary cache substituter. It corresponds to `curl`’s
+          `--connect-timeout` option.
+        )"};
 
-    Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
-        "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
+    Setting<unsigned long> stalledDownloadTimeout{
+        this, 300, "stalled-download-timeout",
+        R"(
+          The timeout (in seconds) for receiving data from servers
+          during download. Nix cancels idle downloads after this
+          timeout's duration.
+        )"};
 
     Setting<unsigned int> tries{this, 5, "download-attempts",
         "How often Nix will attempt to download a file before giving up."};
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index e3bb4cf84..ab9f42ce6 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -80,89 +80,209 @@ public:
     Setting<bool> keepGoing{this, false, "keep-going",
         "Whether to keep building derivations when another build fails."};
 
-    Setting<bool> tryFallback{this, false, "fallback",
-        "Whether to fall back to building when substitution fails.",
+    Setting<bool> tryFallback{
+        this, false, "fallback",
+        R"(
+          If set to `true`, Nix will fall back to building from source if a
+          binary substitute fails. This is equivalent to the `--fallback`
+          flag. The default is `false`.
+        )",
         {"build-fallback"}};
 
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
 
     Setting<size_t> logLines{this, 10, "log-lines",
-        "If verbose-build is false, the number of lines of the tail of "
+        "If `verbose-build` is false, the number of lines of the tail of "
         "the log to show if a build fails."};
 
-    MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
-        "Maximum number of parallel build jobs. \"auto\" means use number of cores.",
+    MaxBuildJobsSetting maxBuildJobs{
+        this, 1, "max-jobs",
+        R"(
+          This option defines the maximum number of jobs that Nix will try to
+          build in parallel. The default is `1`. The special value `auto`
+          causes Nix to use the number of CPUs in your system. `0` is useful
+          when using remote builders to prevent any local builds (except for
+          `preferLocalBuild` derivation attribute which executes locally
+          regardless). It can be overridden using the `--max-jobs` (`-j`)
+          command line switch.
+        )",
         {"build-max-jobs"}};
 
-    Setting<unsigned int> buildCores{this, getDefaultCores(), "cores",
-        "Number of CPU cores to utilize in parallel within a build, "
-        "i.e. by passing this number to Make via '-j'. 0 means that the "
-        "number of actual CPU cores on the local host ought to be "
-        "auto-detected.", {"build-cores"}};
+    Setting<unsigned int> buildCores{
+        this, getDefaultCores(), "cores",
+        R"(
+          Sets the value of the `NIX_BUILD_CORES` environment variable in the
+          invocation of builders. Builders can use this variable at their
+          discretion to control the maximum amount of parallelism. For
+          instance, in Nixpkgs, if the derivation attribute
+          `enableParallelBuilding` is set to `true`, the builder passes the
+          `-jN` flag to GNU Make. It can be overridden using the `--cores`
+          command line switch and defaults to `1`. The value `0` means that
+          the builder should use all available CPU cores in the system.
+        )",
+        {"build-cores"}};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
     bool readOnlyMode = false;
 
-    Setting<std::string> thisSystem{this, SYSTEM, "system",
-        "The canonical Nix system name."};
+    Setting<std::string> thisSystem{
+        this, SYSTEM, "system",
+        R"(
+          This option specifies the canonical Nix system name of the current
+          installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
+          build derivations whose `system` attribute equals the value
+          specified here. In general, it never makes sense to modify this
+          value from its default, since you can use it to ‘lie’ about the
+          platform you are building on (e.g., perform a Mac OS build on a
+          Linux machine; the result would obviously be wrong). It only makes
+          sense if the Nix binaries can run on multiple platforms, e.g.,
+          ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
 
-    Setting<time_t> maxSilentTime{this, 0, "max-silent-time",
-        "The maximum time in seconds that a builer can go without "
-        "producing any output on stdout/stderr before it is killed. "
-        "0 means infinity.",
+          It defaults to the canonical Nix system name detected by `configure`
+          at build time.
+        )"};
+
+    Setting<time_t> maxSilentTime{
+        this, 0, "max-silent-time",
+        R"(
+          This option defines the maximum number of seconds that a builder can
+          go without producing any data on standard output or standard error.
+          This is useful (for instance in an automated build system) to catch
+          builds that are stuck in an infinite loop, or to catch remote builds
+          that are hanging due to network problems. It can be overridden using
+          the `--max-silent-time` command line switch.
+
+          The value `0` means that there is no timeout. This is also the
+          default.
+        )",
         {"build-max-silent-time"}};
 
-    Setting<time_t> buildTimeout{this, 0, "timeout",
-        "The maximum duration in seconds that a builder can run. "
-        "0 means infinity.", {"build-timeout"}};
+    Setting<time_t> buildTimeout{
+        this, 0, "timeout",
+        R"(
+          This option defines the maximum number of seconds that a builder can
+          run. This is useful (for instance in an automated build system) to
+          catch builds that are stuck in an infinite loop but keep writing to
+          their standard output or standard error. It can be overridden using
+          the `--timeout` command line switch.
+
+          The value `0` means that there is no timeout. This is also the
+          default.
+        )",
+        {"build-timeout"}};
 
     PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
-    Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
-        "A semicolon-separated list of build machines, in the format of nix.machines."};
+    Setting<std::string> builders{
+        this, "@" + nixConfDir + "/machines", "builders",
+        "A semicolon-separated list of build machines, in the format of `nix.machines`."};
 
-    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
-        "Whether build machines should use their own substitutes for obtaining "
-        "build dependencies if possible, rather than waiting for this host to "
-        "upload them."};
+    Setting<bool> buildersUseSubstitutes{
+        this, false, "builders-use-substitutes",
+        R"(
+          If set to `true`, Nix will instruct remote build machines to use
+          their own binary substitutes if available. In practical terms, this
+          means that remote hosts will fetch as many build dependencies as
+          possible from their own substitutes (e.g, from `cache.nixos.org`),
+          instead of waiting for this host to upload them all. This can
+          drastically reduce build times if the network connection between
+          this computer and the remote build host is slow.
+        )"};
 
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
-    Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
-        "Whether SQLite should use fsync()."};
+    Setting<bool> fsyncMetadata{
+        this, true, "fsync-metadata",
+        R"(
+          If set to `true`, changes to the Nix store metadata (in
+          `/nix/var/nix/db`) are synchronously flushed to disk. This improves
+          robustness in case of system crashes, but reduces performance. The
+          default is `true`.
+        )"};
 
     Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
         "Whether SQLite should use WAL mode."};
 
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
-        "Whether to call sync() before registering a path as valid."};
+        "Whether to call `sync()` before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "substitute",
-        "Whether to use substitutes.",
+    Setting<bool> useSubstitutes{
+        this, true, "substitute",
+        R"(
+          If set to `true` (default), Nix will use binary substitutes if
+          available. This option can be disabled to force building from
+          source.
+        )",
         {"build-use-substitutes"}};
 
-    Setting<std::string> buildUsersGroup{this, "", "build-users-group",
-        "The Unix group that contains the build users."};
+    Setting<std::string> buildUsersGroup{
+        this, "", "build-users-group",
+        R"(
+          This options specifies the Unix group containing the Nix build user
+          accounts. In multi-user Nix installations, builds should not be
+          performed by the Nix account since that would allow users to
+          arbitrarily modify the Nix store and database by supplying specially
+          crafted builders; and they cannot be performed by the calling user
+          since that would allow him/her to influence the build result.
+
+          Therefore, if this option is non-empty and specifies a valid group,
+          builds will be performed under the user accounts that are a member
+          of the group specified here (as listed in `/etc/group`). Those user
+          accounts should not be used for any other purpose\!
+
+          Nix will never run two builds under the same user account at the
+          same time. This is to prevent an obvious security hole: a malicious
+          user writing a Nix expression that modifies the build result of a
+          legitimate Nix expression being built by another user. Therefore it
+          is good to have as many Nix build user accounts as you can spare.
+          (Remember: uids are cheap.)
+
+          The build users should have permission to create files in the Nix
+          store, but not delete them. Therefore, `/nix/store` should be owned
+          by the Nix account, its group should be the group specified here,
+          and its mode should be `1775`.
+
+          If the build users group is empty, builds will be performed under
+          the uid of the Nix process (that is, the uid of the caller if
+          `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
+          `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
+          multi-user settings with untrusted users.
+        )"};
 
     Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
         {"build-impersonate-linux-26"}};
 
-    Setting<bool> keepLog{this, true, "keep-build-log",
-        "Whether to store build logs.",
+    Setting<bool> keepLog{
+        this, true, "keep-build-log",
+        R"(
+          If set to `true` (the default), Nix will write the build log of a
+          derivation (i.e. the standard output and error of its builder) to
+          the directory `/nix/var/log/nix/drvs`. The build log can be
+          retrieved using the command `nix-store -l path`.
+        )",
         {"build-keep-log"}};
 
-    Setting<bool> compressLog{this, true, "compress-build-log",
-        "Whether to compress logs.",
+    Setting<bool> compressLog{
+        this, true, "compress-build-log",
+        R"(
+          If set to `true` (the default), build logs written to
+          `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2.
+          Otherwise, they will not be compressed.
+        )",
         {"build-compress-log"}};
 
-    Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
-        "Maximum number of bytes a builder can write to stdout/stderr "
-        "before being killed (0 means no limit).",
+    Setting<unsigned long> maxLogSize{
+        this, 0, "max-build-log-size",
+        R"(
+          This option defines the maximum number of bytes that a builder can
+          write to its stdout/stderr. If the builder exceeds this limit, it’s
+          killed. A value of `0` (the default) means that there is no limit.
+        )",
         {"build-max-log-size"}};
 
     /* When buildRepeat > 0 and verboseBuild == true, whether to print
@@ -177,53 +297,156 @@ public:
         "Whether to check if new GC roots can in fact be found by the "
         "garbage collector."};
 
-    Setting<bool> gcKeepOutputs{this, false, "keep-outputs",
-        "Whether the garbage collector should keep outputs of live derivations.",
+    Setting<bool> gcKeepOutputs{
+        this, false, "keep-outputs",
+        R"(
+          If `true`, the garbage collector will keep the outputs of
+          non-garbage derivations. If `false` (default), outputs will be
+          deleted unless they are GC roots themselves (or reachable from other
+          roots).
+
+          In general, outputs must be registered as roots separately. However,
+          even if the output of a derivation is registered as a root, the
+          collector will still delete store paths that are used only at build
+          time (e.g., the C compiler, or source tarballs downloaded from the
+          network). To prevent it from doing so, set this option to `true`.
+        )",
         {"gc-keep-outputs"}};
 
-    Setting<bool> gcKeepDerivations{this, true, "keep-derivations",
-        "Whether the garbage collector should keep derivers of live paths.",
+    Setting<bool> gcKeepDerivations{
+        this, true, "keep-derivations",
+        R"(
+          If `true` (default), the garbage collector will keep the derivations
+          from which non-garbage store paths were built. If `false`, they will
+          be deleted unless explicitly registered as a root (or reachable from
+          other roots).
+
+          Keeping derivation around is useful for querying and traceability
+          (e.g., it allows you to ask with what dependencies or options a
+          store path was built), so by default this option is on. Turn it off
+          to save a bit of disk space (or a lot if `keep-outputs` is also
+          turned on).
+        )",
         {"gc-keep-derivations"}};
 
-    Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
-        "Whether to automatically replace files with identical contents with hard links."};
+    Setting<bool> autoOptimiseStore{
+        this, false, "auto-optimise-store",
+        R"(
+          If set to `true`, Nix automatically detects files in the store
+          that have identical contents, and replaces them with hard links to
+          a single copy. This saves disk space. If set to `false` (the
+          default), you can still run `nix-store --optimise` to get rid of
+          duplicate files.
+        )"};
 
-    Setting<bool> envKeepDerivations{this, false, "keep-env-derivations",
-        "Whether to add derivations as a dependency of user environments "
-        "(to prevent them from being GCed).",
+    Setting<bool> envKeepDerivations{
+        this, false, "keep-env-derivations",
+        R"(
+          If `false` (default), derivations are not stored in Nix user
+          environments. That is, the derivations of any build-time-only
+          dependencies may be garbage-collected.
+
+          If `true`, when you add a Nix derivation to a user environment, the
+          path of the derivation is stored in the user environment. Thus, the
+          derivation will not be garbage-collected until the user environment
+          generation is deleted (`nix-env --delete-generations`). To prevent
+          build-time-only dependencies from being collected, you should also
+          turn on `keep-outputs`.
+
+          The difference between this option and `keep-derivations` is that
+          this one is “sticky”: it applies to any user environment created
+          while this option was enabled, while `keep-derivations` only applies
+          at the moment the garbage collector is run.
+        )",
         {"env-keep-derivations"}};
 
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
-    Setting<SandboxMode> sandboxMode{this,
+    Setting<SandboxMode> sandboxMode{
+        this,
         #if __linux__
           smEnabled
         #else
           smDisabled
         #endif
         , "sandbox",
-        "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
+        R"(
+          If set to `true`, builds will be performed in a *sandboxed
+          environment*, i.e., they’re isolated from the normal file system
+          hierarchy and will only see their dependencies in the Nix store,
+          the temporary build directory, private versions of `/proc`,
+          `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths
+          configured with the `sandbox-paths` option. This is useful to
+          prevent undeclared dependencies on files in directories such as
+          `/usr/bin`. In addition, on Linux, builds run in private PID,
+          mount, network, IPC and UTS namespaces to isolate them from other
+          processes in the system (except that fixed-output derivations do
+          not run in private network namespace to ensure they can access the
+          network).
+
+          Currently, sandboxing only work on Linux and macOS. The use of a
+          sandbox requires that Nix is run as root (so you should use the
+          “build users” feature to perform the actual builds under different
+          users than root).
+
+          If this option is set to `relaxed`, then fixed-output derivations
+          and derivations that have the `__noChroot` attribute set to `true`
+          do not run in sandboxes.
+
+          The default is `true` on Linux and `false` on all other platforms.
+        )",
         {"build-use-chroot", "build-use-sandbox"}};
 
-    Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths",
-        "The paths to make available inside the build sandbox.",
+    Setting<PathSet> sandboxPaths{
+        this, {}, "sandbox-paths",
+        R"(
+          A list of paths bind-mounted into Nix sandbox environments. You can
+          use the syntax `target=source` to mount a path in a different
+          location in the sandbox; for instance, `/bin=/nix-bin` will mount
+          the path `/nix-bin` as `/bin` inside the sandbox. If *source* is
+          followed by `?`, then it is not an error if *source* does not exist;
+          for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
+          only be mounted in the sandbox if it exists in the host filesystem.
+
+          Depending on how Nix was built, the default value for this option
+          may be empty or provide `/bin/sh` as a bind-mount of `bash`.
+        )",
         {"build-chroot-dirs", "build-sandbox-paths"}};
 
     Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
         "Whether to disable sandboxing when the kernel doesn't allow it."};
 
-    Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
-        "Additional paths to make available inside the build sandbox.",
+    Setting<PathSet> extraSandboxPaths{
+        this, {}, "extra-sandbox-paths",
+        R"(
+          A list of additional paths appended to `sandbox-paths`. Useful if
+          you want to extend its default value.
+        )",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
 
-    Setting<size_t> buildRepeat{this, 0, "repeat",
-        "The number of times to repeat a build in order to verify determinism.",
+    Setting<size_t> buildRepeat{
+        this, 0, "repeat",
+        R"(
+          How many times to repeat builds to check whether they are
+          deterministic. The default value is 0. If the value is non-zero,
+          every build is repeated the specified number of times. If the
+          contents of any of the runs differs from the previous ones and
+          `enforce-determinism` is true, the build is rejected and the
+          resulting store paths are not registered as “valid” in Nix’s
+          database.
+        )",
         {"build-repeat"}};
 
 #if __linux__
-    Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
-        "The size of /dev/shm in the build sandbox."};
+    Setting<std::string> sandboxShmSize{
+        this, "50%", "sandbox-dev-shm-size",
+        R"(
+          This option determines the maximum size of the `tmpfs` filesystem
+          mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+          description of the `size` option of `tmpfs` in mount8. The default
+          is `50%`.
+        )"};
 
     Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
         "The build directory inside the sandbox."};
@@ -237,121 +460,411 @@ public:
         "Whether to log Darwin sandbox access violations to the system log."};
 #endif
 
-    Setting<bool> runDiffHook{this, false, "run-diff-hook",
-        "Whether to run the program specified by the diff-hook setting "
-        "repeated builds produce a different result. Typically used to "
-        "plug in diffoscope."};
+    Setting<bool> runDiffHook{
+        this, false, "run-diff-hook",
+        R"(
+          If true, enable the execution of the `diff-hook` program.
 
-    PathSetting diffHook{this, true, "", "diff-hook",
-        "A program that prints out the differences between the two paths "
-        "specified on its command line."};
+          When using the Nix daemon, `run-diff-hook` must be set in the
+          `nix.conf` configuration file, and cannot be passed at the command
+          line.
+        )"};
 
-    Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
-        "Whether to fail if repeated builds produce different output."};
+    PathSetting diffHook{
+        this, true, "", "diff-hook",
+        R"(
+          Absolute path to an executable capable of diffing build
+          results. The hook is executed if `run-diff-hook` is true, and the
+          output of a build is known to not be the same. This program is not
+          executed to determine if two results are the same.
 
-    Setting<Strings> trustedPublicKeys{this,
+          The diff hook is executed by the same user and group who ran the
+          build. However, the diff hook does not have write access to the
+          store path just built.
+
+          The diff hook program receives three parameters:
+
+          1.  A path to the previous build's results
+
+          2.  A path to the current build's results
+
+          3.  The path to the build's derivation
+
+          4.  The path to the build's scratch directory. This directory will
+              exist only if the build was run with `--keep-failed`.
+
+          The stderr and stdout output from the diff hook will not be
+          displayed to the user. Instead, it will print to the nix-daemon's
+          log.
+
+          When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
+          configuration file, and cannot be passed at the command line.
+        )"};
+
+    Setting<bool> enforceDeterminism{
+        this, true, "enforce-determinism",
+        "Whether to fail if repeated builds produce different output. See `repeat`."};
+
+    Setting<Strings> trustedPublicKeys{
+        this,
         {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
         "trusted-public-keys",
-        "Trusted public keys for secure substitution.",
+        R"(
+          A whitespace-separated list of public keys. When paths are copied
+          from another Nix store (such as a binary cache), they must be
+          signed with one of these keys. For example:
+          `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+          hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
+        )",
         {"binary-cache-public-keys"}};
 
-    Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
-        "Secret keys with which to sign local builds."};
+    Setting<Strings> secretKeyFiles{
+        this, {}, "secret-key-files",
+        R"(
+          A whitespace-separated list of files containing secret (private)
+          keys. These are used to sign locally-built paths. They can be
+          generated using `nix-store --generate-binary-cache-key`. The
+          corresponding public key can be distributed to other users, who
+          can add it to `trusted-public-keys` in their `nix.conf`.
+        )"};
 
-    Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
-        "How long downloaded files are considered up-to-date."};
+    Setting<unsigned int> tarballTtl{
+        this, 60 * 60, "tarball-ttl",
+        R"(
+          The number of seconds a downloaded tarball is considered fresh. If
+          the cached tarball is stale, Nix will check whether it is still up
+          to date using the ETag header. Nix will download a new version if
+          the ETag header is unsupported, or the cached ETag doesn't match.
 
-    Setting<bool> requireSigs{this, true, "require-sigs",
-        "Whether to check that any non-content-addressed path added to the "
-        "Nix store has a valid signature (that is, one signed using a key "
-        "listed in 'trusted-public-keys'."};
+          Setting the TTL to `0` forces Nix to always check if the tarball is
+          up to date.
 
-    Setting<StringSet> extraPlatforms{this,
+          Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
+
+          Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
+          `fetchTarball`, and `fetchurl` respect this TTL.
+        )"};
+
+    Setting<bool> requireSigs{
+        this, true, "require-sigs",
+        R"(
+          If set to `true` (the default), any non-content-addressed path added
+          or copied to the Nix store (e.g. when substituting from a binary
+          cache) must have a valid signature, that is, be signed using one of
+          the keys listed in `trusted-public-keys` or `secret-key-files`. Set
+          to `false` to disable signature checking.
+        )"};
+
+    Setting<StringSet> extraPlatforms{
+        this,
         std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
-        "Additional platforms that can be built on the local system. "
-        "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
-        "or using hacks like qemu-user."};
+        R"(
+          Platforms other than the native one which this machine is capable of
+          building for. This can be useful for supporting additional
+          architectures on compatible machines: i686-linux can be built on
+          x86\_64-linux machines (and the default for this setting reflects
+          this); armv7 is backwards-compatible with armv6 and armv5tel; some
+          aarch64 machines can also natively run 32-bit ARM code; and
+          qemu-user may be used to support non-native platforms (though this
+          may be slow and buggy). Most values for this are not enabled by
+          default because build systems will often misdetect the target
+          platform and generate incompatible code, so you may wish to
+          cross-check the results of using this option against proper
+          natively-built versions of your derivations.
+        )"};
 
-    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
+    Setting<StringSet> systemFeatures{
+        this, getDefaultSystemFeatures(),
         "system-features",
-        "Optional features that this system implements (like \"kvm\")."};
+        R"(
+          A set of system “features” supported by this machine, e.g. `kvm`.
+          Derivations can express a dependency on such features through the
+          derivation attribute `requiredSystemFeatures`. For example, the
+          attribute
 
-    Setting<Strings> substituters{this,
+              requiredSystemFeatures = [ "kvm" ];
+
+          ensures that the derivation can only be built on a machine with the
+          `kvm` feature.
+
+          This setting by default includes `kvm` if `/dev/kvm` is accessible,
+          and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
+          that are used in Nixpkgs to route builds to specific machines.
+        )"};
+
+    Setting<Strings> substituters{
+        this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",
-        "The URIs of substituters (such as https://cache.nixos.org/).",
+        R"(
+          A list of URLs of substituters, separated by whitespace. The default
+          is `https://cache.nixos.org`.
+        )",
         {"binary-caches"}};
 
     // FIXME: provide a way to add to option values.
-    Setting<Strings> extraSubstituters{this, {}, "extra-substituters",
-        "Additional URIs of substituters.",
+    Setting<Strings> extraSubstituters{
+        this, {}, "extra-substituters",
+        R"(
+          Additional binary caches appended to those specified in
+          `substituters`. When used by unprivileged users, untrusted
+          substituters (i.e. those not listed in `trusted-substituters`) are
+          silently ignored.
+        )",
         {"extra-binary-caches"}};
 
-    Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters",
-        "Disabled substituters that may be enabled via the substituters option by untrusted users.",
+    Setting<StringSet> trustedSubstituters{
+        this, {}, "trusted-substituters",
+        R"(
+          A list of URLs of substituters, separated by whitespace. These are
+          not used by default, but can be enabled by users of the Nix daemon
+          by specifying `--option substituters urls` on the command
+          line. Unprivileged users are only allowed to pass a subset of the
+          URLs listed in `substituters` and `trusted-substituters`.
+        )",
         {"trusted-binary-caches"}};
 
-    Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
-        "Which users or groups are trusted to ask the daemon to do unsafe things."};
+    Setting<Strings> trustedUsers{
+        this, {"root"}, "trusted-users",
+        R"(
+          A list of names of users (separated by whitespace) that have
+          additional rights when connecting to the Nix daemon, such as the
+          ability to specify additional binary caches, or to import unsigned
+          NARs. You can also specify groups by prefixing them with `@`; for
+          instance, `@wheel` means all users in the `wheel` group. The default
+          is `root`.
 
-    Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl",
-        "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
-        "return an invalid path result"};
+          > **Warning**
+          > 
+          > Adding a user to `trusted-users` is essentially equivalent to
+          > giving that user root access to the system. For example, the user
+          > can set `sandbox-paths` and thereby obtain read access to
+          > directories that are otherwise inacessible to them.
+        )"};
 
-    Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
-        "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
-        "return a valid path result."};
+    Setting<unsigned int> ttlNegativeNarInfoCache{
+        this, 3600, "narinfo-cache-negative-ttl",
+        R"(
+          The TTL in seconds for negative lookups. If a store path is queried
+          from a substituter but was not found, there will be a negative
+          lookup cached in the local disk cache database for the specified
+          duration.
+        )"};
+
+    Setting<unsigned int> ttlPositiveNarInfoCache{
+        this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
+        R"(
+          The TTL in seconds for positive lookups. If a store path is queried
+          from a substituter, the result of the query will be cached in the
+          local disk cache database including some of the NAR metadata. The
+          default TTL is a month, setting a shorter TTL for positive lookups
+          can be useful for binary caches that have frequent garbage
+          collection, in which case having a more frequent cache invalidation
+          would prevent trying to pull the path again and failing with a hash
+          mismatch if the build isn't reproducible.
+        )"};
 
     /* ?Who we trust to use the daemon in safe ways */
-    Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
-        "Which users or groups are allowed to connect to the daemon."};
+    Setting<Strings> allowedUsers{
+        this, {"*"}, "allowed-users",
+        R"(
+          A list of names of users (separated by whitespace) that are allowed
+          to connect to the Nix daemon. As with the `trusted-users` option,
+          you can specify groups by prefixing them with `@`. Also, you can
+          allow all users by specifying `*`. The default is `*`.
+
+          Note that trusted users are always allowed to connect.
+        )"};
 
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
-    Setting<std::string> preBuildHook{this, "",
-        "pre-build-hook",
-        "A program to run just before a build to set derivation-specific build settings."};
+    Setting<std::string> preBuildHook{
+        this, "", "pre-build-hook",
+        R"(
+          If set, the path to a program that can set extra derivation-specific
+          settings for this system. This is used for settings that can't be
+          captured by the derivation model itself and are too variable between
+          different versions of the same system to be hard-coded into nix.
 
-    Setting<std::string> postBuildHook{this, "", "post-build-hook",
-        "A program to run just after each successful build."};
+          The hook is passed the derivation path and, if sandboxes are
+          enabled, the sandbox directory. It can then modify the sandbox and
+          send a series of commands to modify various settings to stdout. The
+          currently recognized commands are:
 
-    Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
-        "Path to the netrc file used to obtain usernames/passwords for downloads."};
+            - `extra-sandbox-paths`  
+              Pass a list of files and directories to be included in the
+              sandbox for this build. One entry per line, terminated by an
+              empty line. Entries have the same format as `sandbox-paths`.
+        )"};
+
+    Setting<std::string> postBuildHook{
+        this, "", "post-build-hook",
+        R"(
+          Optional. The path to a program to execute after each build.
+
+          This option is only settable in the global `nix.conf`, or on the
+          command line by trusted users.
+
+          When using the nix-daemon, the daemon executes the hook as `root`.
+          If the nix-daemon is not involved, the hook runs as the user
+          executing the nix-build.
+
+            - The hook executes after an evaluation-time build.
+
+            - The hook does not execute on substituted paths.
+
+            - The hook's output always goes to the user's terminal.
+
+            - If the hook fails, the build succeeds but no further builds
+              execute.
+
+            - The hook executes synchronously, and blocks other builds from
+              progressing while it runs.
+
+          The program executes with no arguments. The program's environment
+          contains the following environment variables:
+
+            - `DRV_PATH`  
+              The derivation for the built paths.
+
+              Example:
+              `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
+
+            - `OUT_PATHS`  
+              Output paths of the built derivation, separated by a space
+              character.
+
+              Example:
+              `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+              /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+              /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+              /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+              /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
+        )"};
+
+    Setting<std::string> netrcFile{
+        this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
+        R"(
+          If set to an absolute path to a `netrc` file, Nix will use the HTTP
+          authentication credentials in this file when trying to download from
+          a remote host through HTTP or HTTPS. Defaults to
+          `$NIX_CONF_DIR/netrc`.
+
+          The `netrc` file consists of a list of accounts in the following
+          format:
+
+              machine my-machine
+              login my-username
+              password my-password
+
+          For the exact syntax, see [the `curl`
+          documentation](https://ec.haxx.se/usingcurl-netrc.html).
+
+          > **Note**
+          > 
+          > This must be an absolute path, and `~` is not resolved. For
+          > example, `~/.netrc` won't resolve to your home directory's
+          > `.netrc`.
+        )"};
 
     /* Path to the SSL CA file used */
     Path caFile;
 
 #if __linux__
-    Setting<bool> filterSyscalls{this, true, "filter-syscalls",
-            "Whether to prevent certain dangerous system calls, such as "
-            "creation of setuid/setgid files or adding ACLs or extended "
-            "attributes. Only disable this if you're aware of the "
-            "security implications."};
+    Setting<bool> filterSyscalls{
+        this, true, "filter-syscalls",
+        R"(
+          Whether to prevent certain dangerous system calls, such as
+          creation of setuid/setgid files or adding ACLs or extended
+          attributes. Only disable this if you're aware of the
+          security implications.
+        )"};
 
-    Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
-        "Whether builders can acquire new privileges by calling programs with "
-        "setuid/setgid bits or with file capabilities."};
+    Setting<bool> allowNewPrivileges{
+        this, false, "allow-new-privileges",
+        R"(
+          (Linux-specific.) By default, builders on Linux cannot acquire new
+          privileges by calling setuid/setgid programs or programs that have
+          file capabilities. For example, programs such as `sudo` or `ping`
+          will fail. (Note that in sandbox builds, no such programs are
+          available unless you bind-mount them into the sandbox via the
+          `sandbox-paths` option.) You can allow the use of such programs by
+          enabling this option. This is impure and usually undesirable, but
+          may be useful in certain scenarios (e.g. to spin up containers or
+          set up userspace network interfaces in tests).
+        )"};
 #endif
 
-    Setting<Strings> hashedMirrors{this, {}, "hashed-mirrors",
-        "A list of servers used by builtins.fetchurl to fetch files by hash."};
+    Setting<Strings> hashedMirrors{
+        this, {}, "hashed-mirrors",
+        R"(
+          A list of web servers used by `builtins.fetchurl` to obtain files by
+          hash. The default is `http://tarballs.nixos.org/`. Given a hash type
+          *ht* and a base-16 hash *h*, Nix will try to download the file from
+          *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
+          they have disappeared from their original URI. For example, given
+          the default mirror `http://tarballs.nixos.org/`, when building the
+          derivation
 
-    Setting<uint64_t> minFree{this, 0, "min-free",
-        "Automatically run the garbage collector when free disk space drops below the specified amount."};
+          ```nix
+          builtins.fetchurl {
+            url = "https://example.org/foo-1.2.3.tar.xz";
+            sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+          }
+          ```
 
-    Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
-        "Stop deleting garbage when free disk space is above the specified amount."};
+          Nix will attempt to download this file from
+          `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
+          first. If it is not available there, if will try the original URI.
+        )"};
+
+    Setting<uint64_t> minFree{
+        this, 0, "min-free",
+        R"(
+          When free disk space in `/nix/store` drops below `min-free` during a
+          build, Nix performs a garbage-collection until `max-free` bytes are
+          available or there is no more garbage. A value of `0` (the default)
+          disables this feature.
+        )"};
+
+    Setting<uint64_t> maxFree{
+        this, std::numeric_limits<uint64_t>::max(), "max-free",
+        R"(
+          When a garbage collection is triggered by the `min-free` option, it
+          stops as soon as `max-free` bytes are available. The default is
+          infinity (i.e. delete all garbage).
+        )"};
 
     Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
         "Number of seconds between checking free disk space."};
 
-    Setting<Paths> pluginFiles{this, {}, "plugin-files",
-        "Plugins to dynamically load at nix initialization time."};
+    Setting<Paths> pluginFiles{
+        this, {}, "plugin-files",
+        R"(
+          A list of plugin files to be loaded by Nix. Each of these files will
+          be dlopened by Nix, allowing them to affect execution through static
+          initialization. In particular, these plugins may construct static
+          instances of RegisterPrimOp to add new primops or constants to the
+          expression language, RegisterStoreImplementation to add new store
+          implementations, RegisterCommand to add new subcommands to the `nix`
+          command, and RegisterSetting to add new nix config settings. See the
+          constructors for those types for more details.
+
+          Since these files are loaded into the same address space as Nix
+          itself, they must be DSOs compatible with the instance of Nix
+          running at the time (i.e. compiled against the same headers, not
+          linked to any incompatible libraries). They should not be linked to
+          any Nix libs directly, as those will be available already at load
+          time.
+
+          If an entry in the list is a directory, all files in the directory
+          are loaded as plugins (non-recursively).
+        )"};
 
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
-        "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+        "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
 
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 8fc700a2b..ff1a71fba 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -149,11 +149,48 @@ void Config::convertToArgs(Args & args, const std::string & category)
             s.second.setting->convertToArg(args, category);
 }
 
+static std::string stripIndentation(std::string_view s)
+{
+    size_t minIndent = 10000;
+    size_t curIndent = 0;
+    bool atStartOfLine = true;
+
+    for (auto & c : s) {
+        if (atStartOfLine && c == ' ')
+            curIndent++;
+        else if (c == '\n') {
+            if (atStartOfLine)
+                minIndent = std::max(minIndent, curIndent);
+            curIndent = 0;
+            atStartOfLine = true;
+        } else {
+            if (atStartOfLine) {
+                minIndent = std::min(minIndent, curIndent);
+                atStartOfLine = false;
+            }
+        }
+    }
+
+    std::string res;
+
+    size_t pos = 0;
+    while (pos < s.size()) {
+        auto eol = s.find('\n', pos);
+        if (eol == s.npos) eol = s.size();
+        if (eol - pos > minIndent)
+            res.append(s.substr(pos + minIndent, eol - pos - minIndent));
+        res.push_back('\n');
+        pos = eol + 1;
+    }
+
+    return res;
+}
+
 AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
     const std::set<std::string> & aliases)
-    : name(name), description(description), aliases(aliases)
+    : name(name), description(stripIndentation(description)), aliases(aliases)
 {
 }
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 09619aac6..63cb2b268 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -37,10 +37,12 @@ typedef uint64_t ActivityId;
 
 struct LoggerSettings : Config
 {
-    Setting<bool> showTrace{this,
-        false,
-        "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
+    Setting<bool> showTrace{
+        this, false, "show-trace",
+        R"(
+          Where Nix should print out a stack trace in case of Nix
+          expression evaluation errors.
+        )"};
 };
 
 extern LoggerSettings loggerSettings;

From c3e20d8c28ac777fff17c56f57ac8aa28c9272b4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Aug 2020 18:30:17 +0200
Subject: [PATCH 116/882] Consistency

---
 doc/manual/generate-options.jq | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/generate-options.jq b/doc/manual/generate-options.jq
index 3ee51ddea..e0cf496d6 100644
--- a/doc/manual/generate-options.jq
+++ b/doc/manual/generate-options.jq
@@ -1,5 +1,5 @@
 . | to_entries | sort_by(.key) | map(
-  "  - `" + .key + "`  \n"
+  "  - `" + .key + "`  \n\n"
   + (.value.description | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
   + "    **Default**: " + (
       if .value.value == "" or .value.value == []

From b4ef3d7078133d6c31c77da45a991bfe2210623a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Aug 2020 21:00:57 +0200
Subject: [PATCH 117/882] Revert "Add a separate manual job"

This reverts commit 5e3ad1dde0a03b3bd094e1d4ecc0f4fc7abdaa5c. Manual
generation now depends on the 'nix' command.
---
 flake.nix | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/flake.nix b/flake.nix
index 8f5a3055c..f97197cce 100644
--- a/flake.nix
+++ b/flake.nix
@@ -144,6 +144,11 @@
 
           installFlags = "sysconfdir=$(out)/etc";
 
+          postInstall = ''
+            mkdir -p $doc/nix-support
+            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+          '';
+
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
@@ -210,25 +215,6 @@
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
 
-        # Separate build for just the manual.
-        manual =
-          with nixpkgsFor.x86_64-linux;
-          stdenv.mkDerivation {
-            name = "nix-manual-${version}";
-            src = self;
-            buildInputs = [ mdbook ];
-            configurePhase = ":";
-            buildPhase = ":";
-            installPhase =
-              ''
-                touch Makefile.config
-                docdir=$out/share/doc/nix
-                make docdir=$docdir doc_generate=yes $docdir/manual/index.html
-                mkdir -p $out/nix-support
-                echo "doc manual $docdir/manual" >> $out/nix-support/hydra-build-products
-              '';
-          };
-
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.

From a83694c7a1212567ec2f032f84c0d72722bcf5ea Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 19 Aug 2020 19:34:47 +0000
Subject: [PATCH 118/882] Use `RemoteStore` to open connection for proxying
 daemon

Removes duplicate websocket opening code, and also means we should be
able to to ssh-ssh-... daemon relays, not just uds-uds-... ones.
---
 src/libstore/remote-store.cc | 13 ++++++++++---
 src/libstore/remote-store.hh |  4 ++--
 src/libstore/ssh-store.cc    |  1 -
 src/nix-daemon/nix-daemon.cc | 36 +++++++++---------------------------
 4 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 553069b89..ff7149643 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -86,7 +86,16 @@ RemoteStore::RemoteStore(const Params & params)
     : Store(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
-            [this]() { return openConnectionWrapper(); },
+            [this]() {
+                auto conn = openConnectionWrapper();
+                try {
+                    initConnection(*conn);
+                } catch (...) {
+                    failed = true;
+                    throw;
+                }
+                return conn;
+            },
             [this](const ref<Connection> & r) {
                 return
                     r->to.good()
@@ -169,8 +178,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
 
     conn->startTime = std::chrono::steady_clock::now();
 
-    initConnection(*conn);
-
     return conn;
 }
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 72d2a6689..5ad2dc75b 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -102,8 +102,6 @@ public:
 
     void flushBadConnections();
 
-protected:
-
     struct Connection
     {
         AutoCloseFD fd;
@@ -119,6 +117,8 @@ protected:
 
     ref<Connection> openConnectionWrapper();
 
+protected:
+
     virtual ref<Connection> openConnection() = 0;
 
     void initConnection(Connection & conn);
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index caae6b596..7e9b44afe 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -89,7 +89,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
         + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
-    initConnection(*conn);
     return conn;
 }
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 9613cb7d3..bdf56d2e2 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -286,46 +286,28 @@ static int _main(int argc, char * * argv)
         initPlugins();
 
         if (stdio) {
-            if (openUncachedStore().dynamic_pointer_cast<UDSRemoteStore>()) {
-                // FIXME Use the connection the UDSRemoteStore opened
+            if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
+                auto conn = store->openConnectionWrapper();
+                int from = conn->from.fd;
+                int to = conn->to.fd;
 
-                //  Forward on this connection to the real daemon
-                auto socketPath = settings.nixDaemonSocketFile;
-                auto s = socket(PF_UNIX, SOCK_STREAM, 0);
-                if (s == -1)
-                    throw SysError("creating Unix domain socket");
-
-                auto socketDir = dirOf(socketPath);
-                if (chdir(socketDir.c_str()) == -1)
-                    throw SysError("changing to socket directory '%1%'", socketDir);
-
-                auto socketName = std::string(baseNameOf(socketPath));
-                auto addr = sockaddr_un{};
-                addr.sun_family = AF_UNIX;
-                if (socketName.size() + 1 >= sizeof(addr.sun_path))
-                    throw Error("socket name %1% is too long", socketName);
-                strcpy(addr.sun_path, socketName.c_str());
-
-                if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-                    throw SysError("cannot connect to daemon at %1%", socketPath);
-
-                auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1;
+                auto nfds = std::max(from, to) + 1;
                 while (true) {
                     fd_set fds;
                     FD_ZERO(&fds);
-                    FD_SET(s, &fds);
+                    FD_SET(from, &fds);
                     FD_SET(STDIN_FILENO, &fds);
                     if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
                         throw SysError("waiting for data from client or server");
-                    if (FD_ISSET(s, &fds)) {
-                        auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+                    if (FD_ISSET(from, &fds)) {
+                        auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
                         if (res == -1)
                             throw SysError("splicing data from daemon socket to stdout");
                         else if (res == 0)
                             throw EndOfFile("unexpected EOF from daemon socket");
                     }
                     if (FD_ISSET(STDIN_FILENO, &fds)) {
-                        auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+                        auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
                         if (res == -1)
                             throw SysError("splicing data from stdin to daemon socket");
                         else if (res == 0)

From f36793c7b950001f80291e162b34f1f9f2152fa0 Mon Sep 17 00:00:00 2001
From: Ryan Mulligan <ryan@ryantm.com>
Date: Wed, 19 Aug 2020 20:31:01 -0700
Subject: [PATCH 119/882] fix spelling

---
 src/nix/develop.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 434088da7..9aaa80822 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -246,7 +246,7 @@ struct CmdDevelop : Common, MixEnvironment
         addFlag({
             .longName = "command",
             .shortName = 'c',
-            .description = "command and arguments to be executed insted of an interactive shell",
+            .description = "command and arguments to be executed instead of an interactive shell",
             .labels = {"command", "args"},
             .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");

From 3df78858f2ad91a80e30ba910119a0c16c05c66a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 20 Aug 2020 05:08:50 +0000
Subject: [PATCH 120/882] Fix max fd calc and add test

---
 src/nix-daemon/nix-daemon.cc |  2 +-
 tests/local.mk               |  1 +
 tests/ssh-relay.sh           | 16 ++++++++++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 tests/ssh-relay.sh

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index bdf56d2e2..6e652ccbf 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -291,7 +291,7 @@ static int _main(int argc, char * * argv)
                 int from = conn->from.fd;
                 int to = conn->to.fd;
 
-                auto nfds = std::max(from, to) + 1;
+                auto nfds = std::max(from, STDIN_FILENO) + 1;
                 while (true) {
                     fd_set fds;
                     FD_ZERO(&fds);
diff --git a/tests/local.mk b/tests/local.mk
index 53035da41..fd9438386 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -15,6 +15,7 @@ nix_tests = \
   linux-sandbox.sh \
   build-dry.sh \
   build-remote-input-addressed.sh \
+  ssh-relay.sh \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
diff --git a/tests/ssh-relay.sh b/tests/ssh-relay.sh
new file mode 100644
index 000000000..dce50974b
--- /dev/null
+++ b/tests/ssh-relay.sh
@@ -0,0 +1,16 @@
+source common.sh
+
+echo foo > $TEST_ROOT/hello.sh
+
+ssh_localhost=ssh://localhost
+remote_store=?remote-store=$ssh_localhost
+
+store=$ssh_localhost
+
+store+=$remote_store
+store+=$remote_store
+store+=$remote_store
+
+out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh)
+
+[ foo = $(< $out) ]

From acb99f03f963d00670e5a286c76a92d022ef44b0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Aug 2020 11:02:16 +0200
Subject: [PATCH 121/882] Config: Use nlohmann/json

---
 precompiled-headers.h   |  2 ++
 src/libstore/globals.cc |  6 ++++--
 src/libutil/config.cc   | 41 +++++++++++++++++++++--------------------
 src/libutil/config.hh   | 14 +++++++-------
 src/nix/show-config.cc  |  6 +++---
 5 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/precompiled-headers.h b/precompiled-headers.h
index 079aa496e..f52f1cab8 100644
--- a/precompiled-headers.h
+++ b/precompiled-headers.h
@@ -56,3 +56,5 @@
 #include <sys/wait.h>
 #include <termios.h>
 #include <unistd.h>
+
+#include <nlohmann/json.hpp>
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 683fa5196..4a5971c3f 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -9,6 +9,8 @@
 #include <dlfcn.h>
 #include <sys/utsname.h>
 
+#include <nlohmann/json.hpp>
+
 
 namespace nix {
 
@@ -160,9 +162,9 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
     else abort();
 }
 
-template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
+template<> nlohmann::json BaseSetting<SandboxMode>::toJSON()
 {
-    AbstractSetting::toJSON(out);
+    return AbstractSetting::toJSON();
 }
 
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index ff1a71fba..4c5b80d49 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -1,6 +1,7 @@
 #include "config.hh"
 #include "args.hh"
-#include "json.hh"
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -131,15 +132,17 @@ void Config::resetOverriden()
         s.second.setting->overriden = false;
 }
 
-void Config::toJSON(JSONObject & out)
+nlohmann::json Config::toJSON()
 {
+    auto res = nlohmann::json::object();
     for (auto & s : _settings)
         if (!s.second.isAlias) {
-            JSONObject out2(out.object(s.first));
-            out2.attr("description", s.second.setting->description);
-            JSONPlaceholder out3(out2.placeholder("value"));
-            s.second.setting->toJSON(out3);
+            auto obj = nlohmann::json::object();
+            obj.emplace("description", s.second.setting->description);
+            obj.emplace("value", s.second.setting->toJSON());
+            res.emplace(s.first, obj);
         }
+    return res;
 }
 
 void Config::convertToArgs(Args & args, const std::string & category)
@@ -199,9 +202,9 @@ void AbstractSetting::setDefault(const std::string & str)
     if (!overriden) set(str);
 }
 
-void AbstractSetting::toJSON(JSONPlaceholder & out)
+nlohmann::json AbstractSetting::toJSON()
 {
-    out.write(to_string());
+    return to_string();
 }
 
 void AbstractSetting::convertToArg(Args & args, const std::string & category)
@@ -209,9 +212,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category)
 }
 
 template<typename T>
-void BaseSetting<T>::toJSON(JSONPlaceholder & out)
+nlohmann::json BaseSetting<T>::toJSON()
 {
-    out.write(value);
+    return value;
 }
 
 template<typename T>
@@ -292,11 +295,9 @@ template<> std::string BaseSetting<Strings>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out)
+template<> nlohmann::json BaseSetting<Strings>::toJSON()
 {
-    JSONList list(out.list());
-    for (auto & s : value)
-        list.elem(s);
+    return value;
 }
 
 template<> void BaseSetting<StringSet>::set(const std::string & str)
@@ -309,11 +310,9 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out)
+template<> nlohmann::json BaseSetting<StringSet>::toJSON()
 {
-    JSONList list(out.list());
-    for (auto & s : value)
-        list.elem(s);
+    return value;
 }
 
 template class BaseSetting<int>;
@@ -360,10 +359,12 @@ void GlobalConfig::resetOverriden()
         config->resetOverriden();
 }
 
-void GlobalConfig::toJSON(JSONObject & out)
+nlohmann::json GlobalConfig::toJSON()
 {
+    auto res = nlohmann::json::object();
     for (auto & config : *configRegistrations)
-        config->toJSON(out);
+        res.update(config->toJSON());
+    return res;
 }
 
 void GlobalConfig::convertToArgs(Args & args, const std::string & category)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 66073546e..2b4265806 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -4,6 +4,8 @@
 
 #include "types.hh"
 
+#include <nlohmann/json_fwd.hpp>
+
 #pragma once
 
 namespace nix {
@@ -42,8 +44,6 @@ namespace nix {
 
 class Args;
 class AbstractSetting;
-class JSONPlaceholder;
-class JSONObject;
 
 class AbstractConfig
 {
@@ -97,7 +97,7 @@ public:
      * Outputs all settings to JSON
      * - out: JSONObject to write the configuration to
      */
-    virtual void toJSON(JSONObject & out) = 0;
+    virtual nlohmann::json toJSON() = 0;
 
     /**
      * Converts settings to `Args` to be used on the command line interface
@@ -167,7 +167,7 @@ public:
 
     void resetOverriden() override;
 
-    void toJSON(JSONObject & out) override;
+    nlohmann::json toJSON() override;
 
     void convertToArgs(Args & args, const std::string & category) override;
 };
@@ -206,7 +206,7 @@ protected:
 
     virtual std::string to_string() const = 0;
 
-    virtual void toJSON(JSONPlaceholder & out);
+    virtual nlohmann::json toJSON();
 
     virtual void convertToArg(Args & args, const std::string & category);
 
@@ -251,7 +251,7 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    void toJSON(JSONPlaceholder & out) override;
+    nlohmann::json toJSON() override;
 };
 
 template<typename T>
@@ -319,7 +319,7 @@ struct GlobalConfig : public AbstractConfig
 
     void resetOverriden() override;
 
-    void toJSON(JSONObject & out) override;
+    nlohmann::json toJSON() override;
 
     void convertToArgs(Args & args, const std::string & category) override;
 
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 4fd8886de..3ed1ad2aa 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -2,7 +2,8 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
-#include "json.hh"
+
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -19,8 +20,7 @@ struct CmdShowConfig : Command, MixJSON
     {
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
-            JSONObject jsonObj(std::cout);
-            globalConfig.toJSON(jsonObj);
+            logger->stdout("%s", globalConfig.toJSON().dump());
         } else {
             std::map<std::string, Config::SettingInfo> settings;
             globalConfig.getSettings(settings);

From 3c4f8c91759ac5ed6a211f8e72b9d4e8438db833 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Aug 2020 11:13:17 +0200
Subject: [PATCH 122/882] List deprecated option aliases in the docs

---
 doc/manual/generate-options.jq | 9 +++++++--
 src/libutil/config.cc          | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/manual/generate-options.jq b/doc/manual/generate-options.jq
index e0cf496d6..ccf62e8ed 100644
--- a/doc/manual/generate-options.jq
+++ b/doc/manual/generate-options.jq
@@ -1,11 +1,16 @@
 . | to_entries | sort_by(.key) | map(
   "  - `" + .key + "`  \n\n"
   + (.value.description | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
-  + "    **Default**: " + (
+  + "    **Default:** " + (
       if .value.value == "" or .value.value == []
       then "*empty*"
       elif (.value.value | type) == "array"
       then "`" + (.value.value | join(" ")) + "`"
-      else "`" + (.value.value | tostring) + "`" end)
+      else "`" + (.value.value | tostring) + "`"
+      end)
   + "\n\n"
+  + (if (.value.aliases | length) > 0
+     then "    **Deprecated alias:** " + (.value.aliases | map("`" + . + "`") | join(", ")) + "\n\n"
+     else ""
+     end)
 ) | join("")
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 4c5b80d49..e5297c653 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -139,6 +139,7 @@ nlohmann::json Config::toJSON()
         if (!s.second.isAlias) {
             auto obj = nlohmann::json::object();
             obj.emplace("description", s.second.setting->description);
+            obj.emplace("aliases", s.second.setting->aliases);
             obj.emplace("value", s.second.setting->toJSON());
             res.emplace(s.first, obj);
         }

From dc2f278c95ce4a73749cbb8221a568201535d46a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Aug 2020 12:21:46 +0200
Subject: [PATCH 123/882] Allow 'nix' subcommands to provide docs in Markdown
 format

---
 doc/manual/generate-manpage.jq |  6 ++++-
 src/libutil/args.cc            |  4 +++-
 src/libutil/args.hh            |  4 ++++
 src/libutil/config.cc          | 37 ------------------------------
 src/libutil/util.cc            | 41 ++++++++++++++++++++++++++++++++++
 src/libutil/util.hh            |  6 +++++
 src/nix/add-to-store.cc        |  8 +++++++
 src/nix/repl.cc                |  2 +-
 8 files changed, 68 insertions(+), 40 deletions(-)

diff --git a/doc/manual/generate-manpage.jq b/doc/manual/generate-manpage.jq
index d3cf1c601..dd632f162 100644
--- a/doc/manual/generate-manpage.jq
+++ b/doc/manual/generate-manpage.jq
@@ -12,7 +12,7 @@ def show_flags:
   ;
 
 def show_synopsis:
-  "`" + .command + "` " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n\n"
+  "`" + .command + "` [*flags*...] " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n\n"
   ;
 
 def show_command:
@@ -21,6 +21,10 @@ def show_command:
   + "`" + .command + "` - " + .def.description + "\n\n"
   + .section + " Synopsis\n\n"
   + ({"command": .command, "args": .def.args} | show_synopsis)
+  + (if .def | has("doc")
+     then .section + " Description\n\n" + .def.doc + "\n\n"
+     else ""
+     end)
   + (if (.def.flags | length) > 0 then
       .section + " Flags\n\n"
       + (.def | show_flags)
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ad83a2414..147602415 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -359,12 +359,14 @@ nlohmann::json Command::toJSON()
     for (auto & example : examples()) {
         auto ex = nlohmann::json::object();
         ex["description"] = example.description;
-        ex["command"] = example.command;
+        ex["command"] = chomp(stripIndentation(example.command));
         exs.push_back(std::move(ex));
     }
 
     auto res = Args::toJSON();
     res["examples"] = std::move(exs);
+    auto s = doc();
+    if (s != "") res.emplace("doc", stripIndentation(s));
     return res;
 }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index c56044f1d..3c1f87f7e 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -22,6 +22,7 @@ public:
 
     virtual void printHelp(const string & programName, std::ostream & out);
 
+    /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
 protected:
@@ -221,6 +222,9 @@ struct Command : virtual Args
     virtual void prepare() { };
     virtual void run() = 0;
 
+    /* Return documentation about this command, in Markdown format. */
+    virtual std::string doc() { return ""; }
+
     struct Example
     {
         std::string description;
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index e5297c653..3cf720bce 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -153,43 +153,6 @@ void Config::convertToArgs(Args & args, const std::string & category)
             s.second.setting->convertToArg(args, category);
 }
 
-static std::string stripIndentation(std::string_view s)
-{
-    size_t minIndent = 10000;
-    size_t curIndent = 0;
-    bool atStartOfLine = true;
-
-    for (auto & c : s) {
-        if (atStartOfLine && c == ' ')
-            curIndent++;
-        else if (c == '\n') {
-            if (atStartOfLine)
-                minIndent = std::max(minIndent, curIndent);
-            curIndent = 0;
-            atStartOfLine = true;
-        } else {
-            if (atStartOfLine) {
-                minIndent = std::min(minIndent, curIndent);
-                atStartOfLine = false;
-            }
-        }
-    }
-
-    std::string res;
-
-    size_t pos = 0;
-    while (pos < s.size()) {
-        auto eol = s.find('\n', pos);
-        if (eol == s.npos) eol = s.size();
-        if (eol - pos > minIndent)
-            res.append(s.substr(pos + minIndent, eol - pos - minIndent));
-        res.push_back('\n');
-        pos = eol + 1;
-    }
-
-    return res;
-}
-
 AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index c0b9698ee..9e7142e01 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1464,6 +1464,47 @@ string base64Decode(std::string_view s)
 }
 
 
+std::string stripIndentation(std::string_view s)
+{
+    size_t minIndent = 10000;
+    size_t curIndent = 0;
+    bool atStartOfLine = true;
+
+    for (auto & c : s) {
+        if (atStartOfLine && c == ' ')
+            curIndent++;
+        else if (c == '\n') {
+            if (atStartOfLine)
+                minIndent = std::max(minIndent, curIndent);
+            curIndent = 0;
+            atStartOfLine = true;
+        } else {
+            if (atStartOfLine) {
+                minIndent = std::min(minIndent, curIndent);
+                atStartOfLine = false;
+            }
+        }
+    }
+
+    std::string res;
+
+    size_t pos = 0;
+    while (pos < s.size()) {
+        auto eol = s.find('\n', pos);
+        if (eol == s.npos) eol = s.size();
+        if (eol - pos > minIndent)
+            res.append(s.substr(pos + minIndent, eol - pos - minIndent));
+        res.push_back('\n');
+        pos = eol + 1;
+    }
+
+    return res;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3a20679a8..082e26375 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -464,6 +464,12 @@ string base64Encode(std::string_view s);
 string base64Decode(std::string_view s);
 
 
+/* Remove common leading whitespace from the lines in the string
+   's'. For example, if every line is indented by at least 3 spaces,
+   then we remove 3 spaces from the start of every line. */
+std::string stripIndentation(std::string_view s);
+
+
 /* Get a value for the specified key from an associate container. */
 template <class T>
 std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 713155840..023ffa4ed 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -36,6 +36,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         return "add a path to the Nix store";
     }
 
+    std::string doc() override
+    {
+        return R"(
+          Copy the file or directory *path* to the Nix store, and
+          print the resulting store path on standard output.
+        )";
+    }
+
     Examples examples() override
     {
         return {
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index a74655200..0cbe9643c 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -782,7 +782,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         return {
           Example{
             "Display all special commands within the REPL:",
-              "nix repl\n  nix-repl> :?"
+            "nix repl\nnix-repl> :?"
           }
         };
     }

From 25ecfffdc322d8df5834e6add1c5e28fcf2818f9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Aug 2020 12:34:04 +0200
Subject: [PATCH 124/882] Remove PrimOp constructor

---
 src/libexpr/eval.cc | 4 ++--
 src/libexpr/eval.hh | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0123070d1..9a155c055 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -509,7 +509,7 @@ Value * EvalState::addPrimOp(const string & name,
     if (arity == 0) {
         auto vPrimOp = allocValue();
         vPrimOp->type = tPrimOp;
-        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
+        vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
@@ -517,7 +517,7 @@ Value * EvalState::addPrimOp(const string & name,
 
     Value * v = allocValue();
     v->type = tPrimOp;
-    v->primOp = new PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 1db3aa766..38e025a3d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -30,8 +30,6 @@ struct PrimOp
     PrimOpFun fun;
     size_t arity;
     Symbol name;
-    PrimOp(PrimOpFun fun, size_t arity, Symbol name)
-        : fun(fun), arity(arity), name(name) { }
 };
 
 

From 8ce88adad9728142e1f11f7e00d2f31925f8fdd1 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Thu, 20 Aug 2020 13:21:22 +0200
Subject: [PATCH 125/882] set Content-Type to "text/plain" for install script

fixes #3947
---
 maintainers/upload-release.pl | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 91ae896d6..6f3882a12 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -117,7 +117,16 @@ for my $fn (glob "$tmpDir/*") {
     my $dstKey = "$releaseDir/" . $name;
     unless (defined $releasesBucket->head_key($dstKey)) {
         print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
-        $releasesBucket->add_key_filename($dstKey, $fn)
+
+        my $configuration = ();
+        $configuration->{content_type} = "application/octet-stream";
+
+        if ($fn =~ /.sha256|.asc|install/) {
+            # Text files
+            $configuration->{content_type} = "text/plain";
+        }
+
+        $releasesBucket->add_key_filename($dstKey, $fn, $configuration)
             or die $releasesBucket->err . ": " . $releasesBucket->errstr;
     }
 }

From 9a9d834dc7bde0a4eafa2f7412e7a2f0df8c3262 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 20 Aug 2020 14:12:51 +0000
Subject: [PATCH 126/882] Rename drv output querying functions

 - `queryDerivationOutputMapAssumeTotal` -> `queryPartialDerivationOutputMap`
 - `queryDerivationOutputMapAssumeTotal` -> `queryDerivationOutputMap`
---
 src/libstore/build.cc        | 4 ++--
 src/libstore/daemon.cc       | 2 +-
 src/libstore/local-store.cc  | 2 +-
 src/libstore/local-store.hh  | 2 +-
 src/libstore/remote-store.cc | 2 +-
 src/libstore/remote-store.hh | 2 +-
 src/libstore/store-api.cc    | 6 +++---
 src/libstore/store-api.hh    | 6 +++---
 src/nix-env/nix-env.cc       | 2 +-
 9 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b47aeff3b..5592b32eb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2756,11 +2756,11 @@ struct RestrictedStore : public LocalFSStore
     void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
     {
         if (!goal.isAllowed(path))
             throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryDerivationOutputMap(path);
+        return next->queryPartialDerivationOutputMap(path);
     }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 9503915eb..0580101a2 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -325,7 +325,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputMap: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        auto outputs = store->queryDerivationOutputMap(path);
+        auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
         worker_proto::write(*store, to, outputs);
         break;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e96091aae..218b56861 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -782,7 +782,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-std::map<std::string, std::optional<StorePath>> LocalStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
     std::map<std::string, std::optional<StorePath>> outputs;
     BasicDerivation drv = readDerivation(path);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 627b1f557..5af12c2b2 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -133,7 +133,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 005415666..3ee907d1a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -474,7 +474,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-std::map<std::string, std::optional<StorePath>> RemoteStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputMap << printStorePath(path);
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 4b093ad3b..b319e774b 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -51,7 +51,7 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e66c04df4..7e016ac2e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -357,8 +357,8 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
-OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path) {
-    auto resp = queryDerivationOutputMap(path);
+OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
+    auto resp = queryPartialDerivationOutputMap(path);
     OutputPathMap result;
     for (auto & [outName, optOutPath] : resp) {
         if (!optOutPath)
@@ -370,7 +370,7 @@ OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path)
 
 StorePathSet Store::queryDerivationOutputs(const StorePath & path)
 {
-    auto outputMap = this->queryDerivationOutputMapAssumeTotal(path);
+    auto outputMap = this->queryDerivationOutputMap(path);
     StorePathSet outputPaths;
     for (auto & i: outputMap) {
         outputPaths.emplace(std::move(i.second));
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9003ab541..68d66be7c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -343,12 +343,12 @@ public:
     /* Query the mapping outputName => outputPath for the given derivation. All
        outputs are mentioned so ones mising the mapping are mapped to
        `std::nullopt`.  */
-    virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path)
-    { unsupported("queryDerivationOutputMap"); }
+    virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path)
+    { unsupported("queryPartialDerivationOutputMap"); }
 
     /* Query the mapping outputName=>outputPath for the given derivation.
        Assume every output has a mapping and throw an exception otherwise. */
-    OutputPathMap queryDerivationOutputMapAssumeTotal(const StorePath & path);
+    OutputPathMap queryDerivationOutputMap(const StorePath & path);
 
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index d36804658..ddd036070 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -381,7 +381,7 @@ static void queryInstSources(EvalState & state,
 
                 if (path.isDerivation()) {
                     elem.setDrvPath(state.store->printStorePath(path));
-                    auto outputs = state.store->queryDerivationOutputMapAssumeTotal(path);
+                    auto outputs = state.store->queryDerivationOutputMap(path);
                     elem.setOutPath(state.store->printStorePath(outputs.at("out")));
                     if (name.size() >= drvExtension.size() &&
                         string(name, name.size() - drvExtension.size()) == drvExtension)

From 45a2f1baaba8dbd4a4fb27b6ab9baee3c2820220 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 20 Aug 2020 18:14:12 +0000
Subject: [PATCH 127/882] Rename drv output querying functions, like master

 - `queryDerivationOutputMapAssumeTotal` -> `queryPartialDerivationOutputMap`
 - `queryDerivationOutputMapAssumeTotal` -> `queryDerivationOutputMap
---
 src/libexpr/primops.cc       |  2 +-
 src/libstore/build.cc        | 26 +++++++++++++-------------
 src/libstore/daemon.cc       |  2 +-
 src/libstore/local-store.cc  |  2 +-
 src/libstore/local-store.hh  |  2 +-
 src/libstore/misc.cc         |  2 +-
 src/libstore/remote-store.cc |  2 +-
 src/libstore/remote-store.hh |  2 +-
 src/libstore/store-api.cc    |  6 +++---
 src/libstore/store-api.hh    |  6 +++---
 src/nix-build/nix-build.cc   |  2 +-
 src/nix-env/nix-env.cc       |  2 +-
 src/nix-store/nix-store.cc   |  2 +-
 src/nix/build.cc             |  2 +-
 src/nix/repl.cc              |  2 +-
 15 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e6391f509..c3d01eea3 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -64,7 +64,7 @@ void EvalState::realiseContext(const PathSet & context)
        paths. */
     if (allowedPaths) {
         for (auto & [drvPath, outputs] : drvs) {
-            auto outputPaths = store->queryDerivationOutputMapAssumeTotal(drvPath);
+            auto outputPaths = store->queryDerivationOutputMap(drvPath);
             for (auto & outputName : outputs) {
                 if (outputPaths.count(outputName) == 0)
                     throw Error("derivation '%s' does not have an output named '%s'",
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index db93d6092..ba28e78c8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1041,8 +1041,8 @@ private:
     /* Wrappers around the corresponding Store methods that first consult the
        derivation.  This is currently needed because when there is no drv file
        there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMapAssumeTotal();
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
+    OutputPathMap queryDerivationOutputMap();
 
     /* Return the set of (in)valid paths. */
     void checkPathValidity();
@@ -1367,7 +1367,7 @@ void DerivationGoal::repairClosure()
        that produced those outputs. */
 
     /* Get the output closure. */
-    auto outputs = queryDerivationOutputMapAssumeTotal();
+    auto outputs = queryDerivationOutputMap();
     StorePathSet outputClosure;
     for (auto & i : outputs) {
         if (!wantOutput(i.first, wantedOutputs)) continue;
@@ -1386,7 +1386,7 @@ void DerivationGoal::repairClosure()
     std::map<StorePath, StorePath> outputsToDrv;
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryDerivationOutputMap(i);
+            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
             for (auto & j : depOutputs)
                 if (j.second)
                     outputsToDrv.insert_or_assign(*j.second, i);
@@ -1457,7 +1457,7 @@ void DerivationGoal::inputsRealised()
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
             assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryDerivationOutputMap(depDrvPath);
+            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
             for (auto & j : wantedDepOutputs) {
                 if (outputs.count(j) > 0) {
                     auto optRealizedInput = outputs.at(j);
@@ -2912,11 +2912,11 @@ struct RestrictedStore : public LocalFSStore
     void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
     {
         if (!goal.isAllowed(path))
             throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryDerivationOutputMap(path);
+        return next->queryPartialDerivationOutputMap(path);
     }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
@@ -2979,7 +2979,7 @@ struct RestrictedStore : public LocalFSStore
 
         for (auto & path : paths) {
             if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMapAssumeTotal(path.path);
+            auto outputs = next->queryDerivationOutputMap(path.path);
             for (auto & output : outputs)
                 if (wantOutput(output.first, path.outputs))
                     newPaths.insert(output.second);
@@ -4548,7 +4548,7 @@ void DerivationGoal::flushLine()
 }
 
 
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryDerivationOutputMap()
+std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
 {
     if (drv->type() != DerivationType::CAFloating) {
         std::map<std::string, std::optional<StorePath>> res;
@@ -4556,11 +4556,11 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryDerivationO
             res.insert_or_assign(name, output.pathOpt(worker.store, drv->name, name));
         return res;
     } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
+        return worker.store.queryPartialDerivationOutputMap(drvPath);
     }
 }
 
-OutputPathMap DerivationGoal::queryDerivationOutputMapAssumeTotal()
+OutputPathMap DerivationGoal::queryDerivationOutputMap()
 {
     if (drv->type() != DerivationType::CAFloating) {
         OutputPathMap res;
@@ -4568,7 +4568,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMapAssumeTotal()
             res.insert_or_assign(name, *output.second);
         return res;
     } else {
-        return worker.store.queryDerivationOutputMapAssumeTotal(drvPath);
+        return worker.store.queryDerivationOutputMap(drvPath);
     }
 }
 
@@ -4576,7 +4576,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMapAssumeTotal()
 void DerivationGoal::checkPathValidity()
 {
     bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryDerivationOutputMap()) {
+    for (auto & i : queryPartialDerivationOutputMap()) {
         InitialOutputStatus status {
             .wanted = wantOutput(i.first, wantedOutputs),
         };
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 046dfbe6e..a4eeebaab 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -325,7 +325,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputMap: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        auto outputs = store->queryDerivationOutputMap(path);
+        auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
         worker_proto::write(*store, to, outputs);
         break;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 634a8046b..0086bb13e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -803,7 +803,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-std::map<std::string, std::optional<StorePath>> LocalStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
     std::map<std::string, std::optional<StorePath>> outputs;
     BasicDerivation drv = readDerivation(path);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index bf4016b13..4d99e5cf6 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -133,7 +133,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 9024fb2bd..da3981696 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -207,7 +207,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             /* true for regular derivations, and CA derivations for which we
                have a trust mapping for all wanted outputs. */
             auto knownOutputPaths = true;
-            for (auto & [outputName, pathOpt] : queryDerivationOutputMap(path.path)) {
+            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
                 if (!pathOpt) {
                     knownOutputPaths = false;
                     break;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1b1260900..adba17c5e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -474,7 +474,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-std::map<std::string, std::optional<StorePath>> RemoteStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputMap << printStorePath(path);
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 4b093ad3b..b319e774b 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -51,7 +51,7 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1f10f0663..09608646e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -366,8 +366,8 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
-OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path) {
-    auto resp = queryDerivationOutputMap(path);
+OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
+    auto resp = queryPartialDerivationOutputMap(path);
     OutputPathMap result;
     for (auto & [outName, optOutPath] : resp) {
         if (!optOutPath)
@@ -379,7 +379,7 @@ OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path)
 
 StorePathSet Store::queryDerivationOutputs(const StorePath & path)
 {
-    auto outputMap = this->queryDerivationOutputMapAssumeTotal(path);
+    auto outputMap = this->queryDerivationOutputMap(path);
     StorePathSet outputPaths;
     for (auto & i: outputMap) {
         outputPaths.emplace(std::move(i.second));
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 58e9b16c6..6583413a6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -348,12 +348,12 @@ public:
     /* Query the mapping outputName => outputPath for the given derivation. All
        outputs are mentioned so ones mising the mapping are mapped to
        `std::nullopt`.  */
-    virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMap(const StorePath & path)
-    { unsupported("queryDerivationOutputMap"); }
+    virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path)
+    { unsupported("queryPartialDerivationOutputMap"); }
 
     /* Query the mapping outputName=>outputPath for the given derivation.
        Assume every output has a mapping and throw an exception otherwise. */
-    OutputPathMap queryDerivationOutputMapAssumeTotal(const StorePath & path);
+    OutputPathMap queryDerivationOutputMap(const StorePath & path);
 
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index be73ea724..ea4b407e7 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -518,7 +518,7 @@ static void _main(int argc, char * * argv)
             if (counter)
                 drvPrefix += fmt("-%d", counter + 1);
 
-            auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(drvPath);
+            auto builtOutputs = store->queryDerivationOutputMap(drvPath);
 
             for (auto & outputName : wantedOutputs) {
                 auto outputPath = builtOutputs.at(outputName);
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index d36804658..ddd036070 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -381,7 +381,7 @@ static void queryInstSources(EvalState & state,
 
                 if (path.isDerivation()) {
                     elem.setDrvPath(state.store->printStorePath(path));
-                    auto outputs = state.store->queryDerivationOutputMapAssumeTotal(path);
+                    auto outputs = state.store->queryDerivationOutputMap(path);
                     elem.setOutPath(state.store->printStorePath(outputs.at("out")));
                     if (name.size() >= drvExtension.size() &&
                         string(name, name.size() - drvExtension.size()) == drvExtension)
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index cfe97f061..ce5de2ae5 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -65,7 +65,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
 
     if (path.path.isDerivation()) {
         if (build) store->buildPaths({path});
-        auto outputPaths = store->queryDerivationOutputMapAssumeTotal(path.path);
+        auto outputPaths = store->queryDerivationOutputMap(path.path);
         Derivation drv = store->derivationFromPath(path.path);
         rootNr++;
 
diff --git a/src/nix/build.cc b/src/nix/build.cc
index a66e8dac4..cb12ee933 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -74,7 +74,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
                             store2->addPermRoot(bo.path, absPath(symlink), true);
                         },
                         [&](BuildableFromDrv bfd) {
-                            auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(bfd.drvPath);
+                            auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
                             for (auto & output : builtOutputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 0224f891d..2bde85cb7 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -488,7 +488,7 @@ bool NixRepl::processLine(string line)
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
             if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                auto outputs = state->store->queryDerivationOutputMapAssumeTotal(state->store->parseStorePath(drvPath));
+                auto outputs = state->store->queryDerivationOutputMap(state->store->parseStorePath(drvPath));
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : outputs)
                     std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second));

From 422affe1027058c7807adec0dac1aee09b65ec4c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 21 Aug 2020 19:19:14 +0000
Subject: [PATCH 128/882] tabs -> spaces

Sorry I let the tab sneak in there in the first place.
---
 src/libstore/remote-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index c2c6eff66..adba17c5e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -74,7 +74,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
 template<>
 std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
 {
-	auto s = readString(from);
+    auto s = readString(from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 

From 3a7b330b64c6ea77e18a0a96aad7fb14947382d9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 21 Aug 2020 19:35:35 +0000
Subject: [PATCH 129/882] "Downstream placeholders" should not be store paths

Insead they should be opaque `/<hash>` like the placeholders we already
have.
---
 src/libexpr/primops.cc      | 6 +++---
 src/libstore/derivations.cc | 8 +++-----
 src/libstore/derivations.hh | 2 +-
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 0ddba6384..d28024639 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -82,13 +82,13 @@ static void mkOutputString(EvalState & state, Value & v,
     auto optOutputPath = o.second.pathOpt(*state.store, drv.name, o.first);
     mkString(
         *state.allocAttr(v, state.symbols.create(o.first)),
-        state.store->printStorePath(optOutputPath
-            ? *optOutputPath
+        optOutputPath
+            ? state.store->printStorePath(*optOutputPath)
             /* Downstream we would substitute this for an actual path once
                we build the floating CA derivation */
             /* FIXME: we need to depend on the basic derivation, not
                derivation */
-            : downstreamPlaceholder(*state.store, drvPath, o.first)),
+            : downstreamPlaceholder(*state.store, drvPath, o.first),
         {"!" + o.first + "!" + state.store->printStorePath(drvPath)});
 }
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index fb1d06466..5319c1a38 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -664,14 +664,12 @@ std::string hashPlaceholder(const std::string & outputName)
     return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
 }
 
-StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
+std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
 {
     auto drvNameWithExtension = drvPath.name();
     auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
-    return store.makeStorePath(
-        "downstream-placeholder:" + std::string { drvPath.name() } + ":" + std::string { outputName },
-        "compressed:" + std::string { drvPath.hashPart() },
-        outputPathName(drvName, outputName));
+    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
+    return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
 }
 
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 49374d046..76f983561 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -196,6 +196,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 
 std::string hashPlaceholder(const std::string & outputName);
 
-StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
+std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
 
 }

From 35e6288be1a2384a29caccd64d88a6b623e6c033 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 23 Aug 2020 15:00:25 +0000
Subject: [PATCH 130/882] `writeDerivation` just needs a plain store reference

---
 src/libexpr/primops.cc      | 2 +-
 src/libstore/derivations.cc | 8 ++++----
 src/libstore/derivations.hh | 2 +-
 src/nix/develop.cc          | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 30f4c3529..dcc34407b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -839,7 +839,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(state.store, drv, state.repair);
+    auto drvPath = writeDerivation(*state.store, drv, state.repair);
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index a9fed2564..43bc61e55 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -61,7 +61,7 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-StorePath writeDerivation(ref<Store> store,
+StorePath writeDerivation(Store & store,
     const Derivation & drv, RepairFlag repair)
 {
     auto references = drv.inputSrcs;
@@ -71,10 +71,10 @@ StorePath writeDerivation(ref<Store> store,
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
     auto suffix = std::string(drv.name) + drvExtension;
-    auto contents = drv.unparse(*store, false);
+    auto contents = drv.unparse(store, false);
     return settings.readOnlyMode
-        ? store->computeStorePathForText(suffix, contents, references)
-        : store->addTextToStore(suffix, contents, references, repair);
+        ? store.computeStorePathForText(suffix, contents, references)
+        : store.addTextToStore(suffix, contents, references, repair);
 }
 
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 3aae30ab2..9656a32f2 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -146,7 +146,7 @@ class Store;
 enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
-StorePath writeDerivation(ref<Store> store,
+StorePath writeDerivation(Store & store,
     const Derivation & drv, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 9aaa80822..8a14e45c9 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -138,7 +138,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
                 .path = shellOutPath
             } });
     drv.env["out"] = store->printStorePath(shellOutPath);
-    auto shellDrvPath2 = writeDerivation(store, drv);
+    auto shellDrvPath2 = writeDerivation(*store, drv);
 
     /* Build the derivation. */
     store->buildPaths({{shellDrvPath2}});

From 88d5c9ec584f000c9a66ad34631fe4eb5c194172 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 10:37:10 +0200
Subject: [PATCH 131/882] Fix tests

---
 src/libutil/tests/config.cc | 31 +++++++++----------------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index 74c59fd31..9465fba3f 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -33,7 +33,7 @@ namespace nix {
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "");
-        ASSERT_EQ(iter->second.description, "description");
+        ASSERT_EQ(iter->second.description, "description\n");
     }
 
     TEST(Config, getDefinedOverridenSettingNotSet) {
@@ -59,7 +59,7 @@ namespace nix {
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "value");
-        ASSERT_EQ(iter->second.description, "description");
+        ASSERT_EQ(iter->second.description, "description\n");
     }
 
     TEST(Config, getDefinedSettingSet2) {
@@ -73,7 +73,7 @@ namespace nix {
         const auto e = settings.find("name-of-the-setting");
         ASSERT_NE(e, settings.end());
         ASSERT_EQ(e->second.value, "value");
-        ASSERT_EQ(e->second.description, "description");
+        ASSERT_EQ(e->second.description, "description\n");
     }
 
     TEST(Config, addSetting) {
@@ -152,29 +152,16 @@ namespace nix {
     }
 
     TEST(Config, toJSONOnEmptyConfig) {
-        std::stringstream out;
-        { // Scoped to force the destructor of JSONObject to write the final `}`
-            JSONObject obj(out);
-            Config config;
-            config.toJSON(obj);
-        }
-
-        ASSERT_EQ(out.str(), "{}");
+        ASSERT_EQ(Config().toJSON().dump(), "{}");
     }
 
     TEST(Config, toJSONOnNonEmptyConfig) {
-        std::stringstream out;
-        { // Scoped to force the destructor of JSONObject to write the final `}`
-            JSONObject obj(out);
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+        setting.assign("value");
 
-            Config config;
-            std::map<std::string, Config::SettingInfo> settings;
-            Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-            setting.assign("value");
-
-            config.toJSON(obj);
-        }
-        ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#");
+        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#");
     }
 
     TEST(Config, setSettingAlias) {

From 33b1679d75f2a3a5dac053431a41897ebf96a3f3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 13:11:56 +0200
Subject: [PATCH 132/882] Allow primops to have Markdown documentation

---
 doc/manual/src/expressions/builtins.md |  3 --
 src/libexpr/eval.cc                    | 28 +++++++++++++++++
 src/libexpr/eval.hh                    |  4 +++
 src/libexpr/primops.cc                 | 43 ++++++++++++++++++++------
 src/libexpr/primops.hh                 |  8 +++--
 src/nix/repl.cc                        | 21 ++++++++++++-
 6 files changed, 92 insertions(+), 15 deletions(-)

diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index 7c4a62f54..c258fb3b3 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -9,9 +9,6 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `abort` *s*; `builtins.abort` *s*  
-    Abort Nix expression evaluation, print error message *s*.
-
   - `builtins.add` *e1* *e2*  
     Return the sum of the numbers *e1* and *e2*.
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 9a155c055..6ac8bbc9f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -525,6 +525,34 @@ Value * EvalState::addPrimOp(const string & name,
 }
 
 
+Value * EvalState::addPrimOp(PrimOp && primOp)
+{
+    /* Hack to make constants lazy: turn them into a application of
+       the primop to a dummy value. */
+    if (primOp.arity == 0) {
+        primOp.arity = 1;
+        auto vPrimOp = allocValue();
+        vPrimOp->type = tPrimOp;
+        vPrimOp->primOp = new PrimOp(std::move(primOp));
+        Value v;
+        mkApp(v, *vPrimOp, *vPrimOp);
+        return addConstant(primOp.name, v);
+    }
+
+    Symbol envName = primOp.name;
+    if (hasPrefix(primOp.name, "__"))
+        primOp.name = symbols.create(std::string(primOp.name, 2));
+
+    Value * v = allocValue();
+    v->type = tPrimOp;
+    v->primOp = new PrimOp(std::move(primOp));
+    staticBaseEnv.vars[envName] = baseEnvDispl;
+    baseEnv.values[baseEnvDispl++] = v;
+    baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
+    return v;
+}
+
+
 Value & EvalState::getBuiltin(const string & name)
 {
     return *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 38e025a3d..db8eb3e16 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -30,6 +30,8 @@ struct PrimOp
     PrimOpFun fun;
     size_t arity;
     Symbol name;
+    std::vector<std::string> args;
+    const char * doc = nullptr;
 };
 
 
@@ -240,6 +242,8 @@ private:
     Value * addPrimOp(const string & name,
         size_t arity, PrimOpFun primOp);
 
+    Value * addPrimOp(PrimOp && primOp);
+
 public:
 
     Value & getBuiltin(const string & name);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4e35dedb0..902a37e6b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -445,12 +445,19 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
 }
 
 
-static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    throw Abort("evaluation aborted with the following error message: '%1%'", s);
-}
+static RegisterPrimOp primop_abort({
+    .name = "abort",
+    .args = {"s"},
+    .doc = R"(
+      Abort Nix expression evaluation and print the error message *s*.
+    )",
+    .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v)
+    {
+        PathSet context;
+        string s = state.coerceToString(pos, *args[0], context);
+        throw Abort("evaluation aborted with the following error message: '%1%'", s);
+    }
+});
 
 
 static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -2238,7 +2245,20 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
     std::optional<std::string> requiredFeature)
 {
     if (!primOps) primOps = new PrimOps;
-    primOps->push_back({name, arity, fun, requiredFeature});
+    primOps->push_back({
+        .name = name,
+        .args = {},
+        .arity = arity,
+        .requiredFeature = std::move(requiredFeature),
+        .fun = fun
+    });
+}
+
+
+RegisterPrimOp::RegisterPrimOp(Info && info)
+{
+    if (!primOps) primOps = new PrimOps;
+    primOps->push_back(std::move(info));
 }
 
 
@@ -2314,7 +2334,6 @@ void EvalState::createBaseEnv()
     addPrimOp("__isBool", 1, prim_isBool);
     addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
-    addPrimOp("abort", 1, prim_abort);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
     addPrimOp("__tryEval", 1, prim_tryEval);
     addPrimOp("__getEnv", 1, prim_getEnv);
@@ -2431,7 +2450,13 @@ void EvalState::createBaseEnv()
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
             if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
-                addPrimOp(primOp.name, primOp.arity, primOp.primOp);
+                addPrimOp({
+                    .fun = primOp.fun,
+                    .arity = std::max(primOp.args.size(), primOp.arity),
+                    .name = symbols.create(primOp.name),
+                    .args = std::move(primOp.args),
+                    .doc = primOp.doc,
+                });
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 75c460ecf..ed5e2ea58 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -10,9 +10,11 @@ struct RegisterPrimOp
     struct Info
     {
         std::string name;
-        size_t arity;
-        PrimOpFun primOp;
+        std::vector<std::string> args;
+        size_t arity = 0;
+        const char * doc;
         std::optional<std::string> requiredFeature;
+        PrimOpFun fun;
     };
 
     typedef std::vector<Info> PrimOps;
@@ -26,6 +28,8 @@ struct RegisterPrimOp
         size_t arity,
         PrimOpFun fun,
         std::optional<std::string> requiredFeature = {});
+
+    RegisterPrimOp(Info && info);
 };
 
 /* These primops are disabled without enableNativeCode, but plugins
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 0cbe9643c..d370ca767 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -416,7 +416,8 @@ bool NixRepl::processLine(string line)
              << "  :r            Reload all files\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n"
-             << "  :u <expr>     Build derivation, then start nix-shell\n";
+             << "  :u <expr>     Build derivation, then start nix-shell\n"
+             << "  :doc <expr>   Show documentation of a builtin function\n";
     }
 
     else if (command == ":a" || command == ":add") {
@@ -509,6 +510,24 @@ bool NixRepl::processLine(string line)
     else if (command == ":q" || command == ":quit")
         return false;
 
+    else if (command == ":doc") {
+        Value v;
+        evalString(arg, v);
+        if (v.type == tPrimOp || v.type == tPrimOpApp) {
+            auto v2 = &v;
+            while (v2->type == tPrimOpApp)
+                v2 = v2->primOpApp.left;
+            if (v2->primOp->doc) {
+                // FIXME: format markdown.
+                if (!v2->primOp->args.empty())
+                    std::cout << fmt("Arguments: %s\n\n", concatStringsSep(" ", v2->primOp->args));
+                std::cout << trim(stripIndentation(v2->primOp->doc)) << "\n";
+            } else
+                throw Error("builtin function '%s' does not have documentation", v2->primOp->name);
+        } else
+            throw Error("value does not have documentation");
+    }
+
     else if (command != "")
         throw Error("unknown command '%1%'", command);
 

From a990f063ff7afc6028ab430170ad23e1285d1a6d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 14:31:10 +0200
Subject: [PATCH 133/882] Move primop docs inline

This makes them available to 'nix repl'.
---
 doc/manual/src/SUMMARY.md                     |    1 +
 .../src/expressions/builtin-constants.md      |   20 +
 doc/manual/src/expressions/builtins.md        |  878 +-----------
 src/libexpr/primops.cc                        | 1178 +++++++++++++++--
 src/libexpr/primops/fetchTree.cc              |  175 ++-
 5 files changed, 1274 insertions(+), 978 deletions(-)
 create mode 100644 doc/manual/src/expressions/builtin-constants.md

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 4089caf8a..8281f683f 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -38,6 +38,7 @@
     - [Operators](expressions/language-operators.md)
     - [Derivations](expressions/derivations.md)
       - [Advanced Attributes](expressions/advanced-attributes.md)
+    - [Built-in Constants](expressions/builtin-constants.md)
     - [Built-in Functions](expressions/builtins.md)
 - [Advanced Topics](advanced-topics/advanced-topics.md)
   - [Remote Builds](advanced-topics/distributed-builds.md)
diff --git a/doc/manual/src/expressions/builtin-constants.md b/doc/manual/src/expressions/builtin-constants.md
new file mode 100644
index 000000000..3345a715b
--- /dev/null
+++ b/doc/manual/src/expressions/builtin-constants.md
@@ -0,0 +1,20 @@
+# Built-in Constants
+
+Here are the constants built into the Nix expression evaluator:
+
+  - `builtins`  
+    The set `builtins` contains all the built-in functions and values.
+    You can use `builtins` to test for the availability of features in
+    the Nix installation, e.g.,
+    
+    ```nix
+    if builtins ? getEnv then builtins.getEnv "PATH" else ""
+    ```
+    
+    This allows a Nix expression to fall back gracefully on older Nix
+    installations that don’t have the desired built-in function.
+
+  - `builtins.currentSystem`  
+    The built-in value `currentSystem` evaluates to the Nix platform
+    identifier for the Nix installation on which the expression is being
+    evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins.md
index c258fb3b3..ae3bb150c 100644
--- a/doc/manual/src/expressions/builtins.md
+++ b/doc/manual/src/expressions/builtins.md
@@ -1,374 +1,20 @@
 # Built-in Functions
 
-This section lists the functions and constants built into the Nix
-expression evaluator. (The built-in function `derivation` is discussed
-above.) Some built-ins, such as `derivation`, are always in scope of
-every Nix expression; you can just access them right away. But to
-prevent polluting the namespace too much, most built-ins are not in
+This section lists the functions built into the Nix expression
+evaluator. (The built-in function `derivation` is discussed above.)
+Some built-ins, such as `derivation`, are always in scope of every Nix
+expression; you can just access them right away. But to prevent
+polluting the namespace too much, most built-ins are not in
 scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `builtins.add` *e1* *e2*  
-    Return the sum of the numbers *e1* and *e2*.
-
-  - `builtins.all` *pred* *list*  
-    Return `true` if the function *pred* returns `true` for all elements
-    of *list*, and `false` otherwise.
-
-  - `builtins.any` *pred* *list*  
-    Return `true` if the function *pred* returns `true` for at least one
-    element of *list*, and `false` otherwise.
-
-  - `builtins.attrNames` *set*  
-    Return the names of the attributes in the set *set* in an
-    alphabetically sorted list. For instance, `builtins.attrNames { y
-    = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`.
-
-  - `builtins.attrValues` *set*  
-    Return the values of the attributes in the set *set* in the order
-    corresponding to the sorted attribute names.
-
-  - `baseNameOf` *s*  
-    Return the *base name* of the string *s*, that is, everything
-    following the final slash in the string. This is similar to the GNU
-    `basename` command.
-
-  - `builtins.bitAnd` *e1* *e2*  
-    Return the bitwise AND of the integers *e1* and *e2*.
-
-  - `builtins.bitOr` *e1* *e2*  
-    Return the bitwise OR of the integers *e1* and *e2*.
-
-  - `builtins.bitXor` *e1* *e2*  
-    Return the bitwise XOR of the integers *e1* and *e2*.
-
-  - `builtins`  
-    The set `builtins` contains all the built-in functions and values.
-    You can use `builtins` to test for the availability of features in
-    the Nix installation, e.g.,
-    
-    ```nix
-    if builtins ? getEnv then builtins.getEnv "PATH" else ""
-    ```
-    
-    This allows a Nix expression to fall back gracefully on older Nix
-    installations that don’t have the desired built-in function.
-
-  - `builtins.compareVersions` *s1* *s2*  
-    Compare two strings representing versions and return `-1` if
-    version *s1* is older than version *s2*, `0` if they are the same,
-    and `1` if *s1* is newer than *s2*. The version comparison
-    algorithm is the same as the one used by [`nix-env
-    -u`](../command-ref/nix-env.md#operation---upgrade).
-
-  - `builtins.concatLists` *lists*  
-    Concatenate a list of lists into a single list.
-
-  - `builtins.concatStringsSep` *separator* *list*  
-    Concatenate a list of strings with a separator between each
-    element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] ==
-    "usr/local/bin"`
-
-  - `builtins.currentSystem`  
-    The built-in value `currentSystem` evaluates to the Nix platform
-    identifier for the Nix installation on which the expression is being
-    evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
-
-  - `builtins.deepSeq` *e1* *e2*  
-    This is like `seq e1 e2`, except that *e1* is evaluated *deeply*:
-    if it’s a list or set, its elements or attributes are also
-    evaluated recursively.
-
   - `derivation` *attrs*; `builtins.derivation` *attrs*  
+
     `derivation` is described in [its own section](derivations.md).
 
-  - `dirOf` *s*; `builtins.dirOf` *s*  
-    Return the directory part of the string *s*, that is, everything
-    before the final slash in the string. This is similar to the GNU
-    `dirname` command.
-
-  - `builtins.div` *e1* *e2*  
-    Return the quotient of the numbers *e1* and *e2*.
-
-  - `builtins.elem` *x* *xs*  
-    Return `true` if a value equal to *x* occurs in the list *xs*, and
-    `false` otherwise.
-
-  - `builtins.elemAt` *xs* *n*  
-    Return element *n* from the list *xs*. Elements are counted starting
-    from 0. A fatal error occurs if the index is out of bounds.
-
-  - `builtins.fetchurl` *url*  
-    Download the specified URL and return the path of the downloaded
-    file. This function is not available if [restricted evaluation
-    mode](../command-ref/conf-file.md) is enabled.
-
-  - `fetchTarball` *url*; `builtins.fetchTarball` *url*  
-    Download the specified URL, unpack it and return the path of the
-    unpacked tree. The file must be a tape archive (`.tar`) compressed
-    with `gzip`, `bzip2` or `xz`. The top-level path component of the
-    files in the tarball is removed, so it is best if the tarball
-    contains a single directory at top level. The typical use of the
-    function is to obtain external Nix expression dependencies, such as
-    a particular version of Nixpkgs, e.g.
-    
-    ```nix
-    with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
-
-    stdenv.mkDerivation { … }
-    ```
-    
-    The fetched tarball is cached for a certain amount of time (1 hour
-    by default) in `~/.cache/nix/tarballs/`. You can change the cache
-    timeout either on the command line with `--option tarball-ttl number
-    of seconds` or in the Nix configuration file with this option: ` 
-    number of seconds to cache `.
-    
-    Note that when obtaining the hash with ` nix-prefetch-url ` the
-    option `--unpack` is required.
-    
-    This function can also verify the contents against a hash. In that
-    case, the function takes a set instead of a URL. The set requires
-    the attribute `url` and the attribute `sha256`, e.g.
-    
-    ```nix
-    with import (fetchTarball {
-      url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
-      sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
-    }) {};
-
-    stdenv.mkDerivation { … }
-    ```
-    
-    This function is not available if [restricted evaluation
-    mode](../command-ref/conf-file.md) is enabled.
-
-  - `builtins.fetchGit` *args*  
-    Fetch a path from git. *args* can be a URL, in which case the HEAD
-    of the repo at that URL is fetched. Otherwise, it can be an
-    attribute with the following attributes (all except `url` optional):
-    
-      - url  
-        The URL of the repo.
-    
-      - name  
-        The name of the directory the repo should be exported to in the
-        store. Defaults to the basename of the URL.
-    
-      - rev  
-        The git revision to fetch. Defaults to the tip of `ref`.
-    
-      - ref  
-        The git ref to look for the requested revision under. This is
-        often a branch or tag name. Defaults to `HEAD`.
-        
-        By default, the `ref` value is prefixed with `refs/heads/`. As
-        of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
-        with `refs/`.
-    
-      - submodules  
-        A Boolean parameter that specifies whether submodules should be
-        checked out. Defaults to `false`.
-    
-    Here are some examples of how to use `fetchGit`.
-    
-      - To fetch a private repository over SSH:
-        
-        ```nix
-        builtins.fetchGit {
-          url = "git@github.com:my-secret/repository.git";
-          ref = "master";
-          rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
-        }
-        ```
-    
-      - To fetch an arbitrary reference:
-        
-        ```nix
-        builtins.fetchGit {
-          url = "https://github.com/NixOS/nix.git";
-          ref = "refs/heads/0.5-release";
-        }
-        ```
-    
-      - If the revision you're looking for is in the default branch of
-        the git repository you don't strictly need to specify the branch
-        name in the `ref` attribute.
-        
-        However, if the revision you're looking for is in a future
-        branch for the non-default branch you will need to specify the
-        the `ref` attribute as well.
-        
-        ```nix
-        builtins.fetchGit {
-          url = "https://github.com/nixos/nix.git";
-          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-          ref = "1.11-maintenance";
-        }
-        ```
-        
-        > **Note**
-        > 
-        > It is nice to always specify the branch which a revision
-        > belongs to. Without the branch being specified, the fetcher
-        > might fail if the default branch changes. Additionally, it can
-        > be confusing to try a commit from a non-default branch and see
-        > the fetch fail. If the branch is specified the fault is much
-        > more obvious.
-    
-      - If the revision you're looking for is in the default branch of
-        the git repository you may omit the `ref` attribute.
-        
-        ```nix
-        builtins.fetchGit {
-          url = "https://github.com/nixos/nix.git";
-          rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
-        }
-        ```
-    
-      - To fetch a specific tag:
-        
-        ```nix
-        builtins.fetchGit {
-          url = "https://github.com/nixos/nix.git";
-          ref = "refs/tags/1.9";
-        }
-        ```
-    
-      - To fetch the latest version of a remote branch:
-        
-        ```nix
-        builtins.fetchGit {
-          url = "ssh://git@github.com/nixos/nix.git";
-          ref = "master";
-        }
-        ```
-        
-        > **Note**
-        > 
-        > Nix will refetch the branch in accordance with
-        > the option `tarball-ttl`.
-        
-        > **Note**
-        > 
-        > This behavior is disabled in *Pure evaluation mode*.
-
-  - `builtins.filter` *f* *xs*  
-    Return a list consisting of the elements of *xs* for which the
-    function *f* returns `true`.
-
-  - `builtins.filterSource` *e1* *e2*  
-    This function allows you to copy sources into the Nix store while
-    filtering certain files. For instance, suppose that you want to use
-    the directory `source-dir` as an input to a Nix expression, e.g.
-    
-    ```nix
-    stdenv.mkDerivation {
-      ...
-      src = ./source-dir;
-    }
-    ```
-    
-    However, if `source-dir` is a Subversion working copy, then all
-    those annoying `.svn` subdirectories will also be copied to the
-    store. Worse, the contents of those directories may change a lot,
-    causing lots of spurious rebuilds. With `filterSource` you can
-    filter out the `.svn` directories:
-    
-    ```nix
-    src = builtins.filterSource
-      (path: type: type != "directory" || baseNameOf path != ".svn")
-      ./source-dir;
-    ```
-    
-    Thus, the first argument *e1* must be a predicate function that is
-    called for each regular file, directory or symlink in the source
-    tree *e2*. If the function returns `true`, the file is copied to the
-    Nix store, otherwise it is omitted. The function is called with two
-    arguments. The first is the full path of the file. The second is a
-    string that identifies the type of the file, which is either
-    `"regular"`, `"directory"`, `"symlink"` or `"unknown"` (for other
-    kinds of files such as device nodes or fifos — but note that those
-    cannot be copied to the Nix store, so if the predicate returns
-    `true` for them, the copy will fail). If you exclude a directory,
-    the entire corresponding subtree of *e2* will be excluded.
-
-  - `builtins.foldl’` *op* *nul* *list*  
-    Reduce a list by applying a binary operator, from left to right,
-    e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
-    ...`. The operator is applied strictly, i.e., its arguments are
-    evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]`
-    evaluates to 6.
-
-  - `builtins.functionArgs` *f*  
-    Return a set containing the names of the formal arguments expected
-    by the function *f*. The value of each attribute is a Boolean
-    denoting whether the corresponding argument has a default value. For
-    instance, `functionArgs ({ x, y ? 123}: ...) = { x = false; y =
-    true; }`.
-    
-    "Formal argument" here refers to the attributes pattern-matched by
-    the function. Plain lambdas are not included, e.g. `functionArgs (x:
-    ...) = { }`.
-
-  - `builtins.fromJSON` *e*  
-    Convert a JSON string to a Nix value. For example,
-    
-    ```nix
-    builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
-    ```
-    
-    returns the value `{ x = [ 1 2 3 ]; y = null; }`.
-
-  - `builtins.genList` *generator* *length*  
-    Generate list of size *length*, with each element *i* equal to the
-    value returned by *generator* `i`. For example,
-    
-    ```nix
-    builtins.genList (x: x * x) 5
-    ```
-    
-    returns the list `[ 0 1 4 9 16 ]`.
-
-  - `builtins.getAttr` *s* *set*  
-    `getAttr` returns the attribute named *s* from *set*. Evaluation
-    aborts if the attribute doesn’t exist. This is a dynamic version of
-    the `.` operator, since *s* is an expression rather than an
-    identifier.
-
-  - `builtins.getEnv` *s*  
-    `getEnv` returns the value of the environment variable *s*, or an
-    empty string if the variable doesn’t exist. This function should be
-    used with care, as it can introduce all sorts of nasty environment
-    dependencies in your Nix expression.
-    
-    `getEnv` is used in Nix Packages to locate the file
-    `~/.nixpkgs/config.nix`, which contains user-local settings for Nix
-    Packages. (That is, it does a `getEnv "HOME"` to locate the user’s
-    home directory.)
-
-  - `builtins.hasAttr` *s* *set*  
-    `hasAttr` returns `true` if *set* has an attribute named *s*, and
-    `false` otherwise. This is a dynamic version of the `?` operator,
-    since *s* is an expression rather than an identifier.
-
-  - `builtins.hashString` *type* *s*  
-    Return a base-16 representation of the cryptographic hash of string
-    *s*. The hash algorithm specified by *type* must be one of `"md5"`,
-    `"sha1"`, `"sha256"` or `"sha512"`.
-
-  - `builtins.hashFile` *type* *p*  
-    Return a base-16 representation of the cryptographic hash of the
-    file at path *p*. The hash algorithm specified by *type* must be one
-    of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`.
-
-  - `builtins.head` *list*  
-    Return the first element of a list; abort evaluation if the argument
-    isn’t a list or is an empty list. You can test whether a list is
-    empty by comparing it with `[]`.
-
   - `import` *path*; `builtins.import` *path*  
+
     Load, parse and return the Nix expression in the file *path*. If
     *path* is a directory, the file ` default.nix ` in that directory
     is loaded. Evaluation aborts if the file doesn’t exist or contains
@@ -376,535 +22,47 @@ For instance, `derivation` is also available as `builtins.derivation`.
     system: you can put any Nix expression (such as a set or a
     function) in a separate file, and use it from Nix expressions in
     other files.
-    
+
     > **Note**
-    > 
+    >
     > Unlike some languages, `import` is a regular function in Nix.
     > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*)
     > are [normal path values](language-values.md).
-    
+
     A Nix expression loaded by `import` must not contain any *free
     variables* (identifiers that are not defined in the Nix expression
     itself and are not built-in). Therefore, it cannot refer to
     variables that are in scope at the call site. For instance, if you
     have a calling expression
-    
+
     ```nix
     rec {
       x = 123;
       y = import ./foo.nix;
     }
     ```
-    
+
     then the following `foo.nix` will give an error:
-    
+
     ```nix
     x + 456
     ```
-    
+
     since `x` is not in scope in `foo.nix`. If you want `x` to be
     available in `foo.nix`, you should pass it as a function argument:
-    
+
     ```nix
     rec {
       x = 123;
       y = import ./foo.nix x;
     }
     ```
-    
+
     and
-    
+
     ```nix
     x: x + 456
     ```
-    
+
     (The function argument doesn’t have to be called `x` in `foo.nix`;
     any name would work.)
-
-  - `builtins.intersectAttrs` *e1* *e2*  
-    Return a set consisting of the attributes in the set *e2* that also
-    exist in the set *e1*.
-
-  - `builtins.isAttrs` *e*  
-    Return `true` if *e* evaluates to a set, and `false` otherwise.
-
-  - `builtins.isList` *e*  
-    Return `true` if *e* evaluates to a list, and `false` otherwise.
-
-  - `builtins.isFunction` *e*  
-    Return `true` if *e* evaluates to a function, and `false` otherwise.
-
-  - `builtins.isString` *e*  
-    Return `true` if *e* evaluates to a string, and `false` otherwise.
-
-  - `builtins.isInt` *e*  
-    Return `true` if *e* evaluates to an int, and `false` otherwise.
-
-  - `builtins.isFloat` *e*  
-    Return `true` if *e* evaluates to a float, and `false` otherwise.
-
-  - `builtins.isBool` *e*  
-    Return `true` if *e* evaluates to a bool, and `false` otherwise.
-
-  - `builtins.isPath` *e*  
-    Return `true` if *e* evaluates to a path, and `false` otherwise.
-
-  - `isNull` *e*; `builtins.isNull` *e*  
-    Return `true` if *e* evaluates to `null`, and `false` otherwise.
-    
-    > **Warning**
-    > 
-    > This function is *deprecated*; just write `e == null` instead.
-
-  - `builtins.length` *e*  
-    Return the length of the list *e*.
-
-  - `builtins.lessThan` *e1* *e2*  
-    Return `true` if the number *e1* is less than the number *e2*, and
-    `false` otherwise. Evaluation aborts if either *e1* or *e2* does not
-    evaluate to a number.
-
-  - `builtins.listToAttrs` *e*  
-    Construct a set from a list specifying the names and values of each
-    attribute. Each element of the list should be a set consisting of a
-    string-valued attribute `name` specifying the name of the attribute,
-    and an attribute `value` specifying its value. Example:
-    
-    ```nix
-    builtins.listToAttrs
-      [ { name = "foo"; value = 123; }
-        { name = "bar"; value = 456; }
-      ]
-    ```
-    
-    evaluates to
-    
-    ```nix
-    { foo = 123; bar = 456; }
-    ```
-
-  - `map` *f* *list*; `builtins.map` *f* *list*  
-    Apply the function *f* to each element in the list *list*. For
-    example,
-    
-    ```nix
-    map (x: "foo" + x) [ "bar" "bla" "abc" ]
-    ```
-    
-    evaluates to `[ "foobar" "foobla" "fooabc" ]`.
-
-  - `builtins.match` *regex* *str*  
-    Returns a list if the [extended POSIX regular
-    expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
-    *regex* matches *str* precisely, otherwise returns `null`. Each item
-    in the list is a regex group.
-    
-    ```nix
-    builtins.match "ab" "abc"
-    ```
-    
-    Evaluates to `null`.
-    
-    ```nix
-    builtins.match "abc" "abc"
-    ```
-    
-    Evaluates to `[ ]`.
-    
-    ```nix
-    builtins.match "a(b)(c)" "abc"
-    ```
-    
-    Evaluates to `[ "b" "c" ]`.
-    
-    ```nix
-    builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
-    ```
-    
-    Evaluates to `[ "foo" ]`.
-
-  - `builtins.mul` *e1* *e2*  
-    Return the product of the numbers *e1* and *e2*.
-
-  - `builtins.parseDrvName` *s*  
-    Split the string *s* into a package name and version. The package
-    name is everything up to but not including the first dash followed
-    by a digit, and the version is everything following that dash. The
-    result is returned in a set `{ name, version }`. Thus,
-    `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name =
-    "nix"; version = "0.12pre12876"; }`.
-
-  - `builtins.path` *args*  
-    An enrichment of the built-in path type, based on the attributes
-    present in *args*. All are optional except `path`:
-    
-      - path  
-        The underlying path.
-    
-      - name  
-        The name of the path when added to the store. This can used to
-        reference paths that have nix-illegal characters in their names,
-        like `@`.
-    
-      - filter  
-        A function of the type expected by `builtins.filterSource`,
-        with the same semantics.
-    
-      - recursive  
-        When `false`, when `path` is added to the store it is with a
-        flat hash, rather than a hash of the NAR serialization of the
-        file. Thus, `path` must refer to a regular file, not a
-        directory. This allows similar behavior to `fetchurl`. Defaults
-        to `true`.
-    
-      - sha256  
-        When provided, this is the expected hash of the file at the
-        path. Evaluation will fail if the hash is incorrect, and
-        providing a hash allows `builtins.path` to be used even when the
-        `pure-eval` nix config option is on.
-
-  - `builtins.pathExists` *path*  
-    Return `true` if the path *path* exists at evaluation time, and
-    `false` otherwise.
-
-  - `builtins.placeholder` *output*  
-    Return a placeholder string for the specified *output* that will be
-    substituted by the corresponding output path at build time. Typical
-    outputs would be `"out"`, `"bin"` or `"dev"`.
-
-  - `builtins.readDir` *path*  
-    Return the contents of the directory *path* as a set mapping
-    directory entries to the corresponding file type. For instance, if
-    directory `A` contains a regular file `B` and another directory
-    `C`, then `builtins.readDir ./A` will return the set
-    
-    ```nix
-    { B = "regular"; C = "directory"; }
-    ```
-    
-    The possible values for the file type are `"regular"`,
-    `"directory"`, `"symlink"` and `"unknown"`.
-
-  - `builtins.readFile` *path*  
-    Return the contents of the file *path* as a string.
-
-  - `removeAttrs` *set* *list*; `builtins.removeAttrs` *set* *list*  
-    Remove the attributes listed in *list* from *set*. The attributes
-    don’t have to exist in *set*. For instance,
-    
-    ```nix
-    removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]
-    ```
-    
-    evaluates to `{ y = 2; }`.
-
-  - `builtins.replaceStrings` *from* *to* *s*  
-    Given string *s*, replace every occurrence of the strings in *from*
-    with the corresponding string in *to*. For example,
-    
-    ```nix
-    builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
-    ```
-    
-    evaluates to `"fabir"`.
-
-  - `builtins.seq` *e1* *e2*  
-    Evaluate *e1*, then evaluate and return *e2*. This ensures that a
-    computation is strict in the value of *e1*.
-
-  - `builtins.sort` *comparator* *list*  
-    Return *list* in sorted order. It repeatedly calls the function
-    *comparator* with two elements. The comparator should return `true`
-    if the first element is less than the second, and `false` otherwise.
-    For example,
-    
-    ```nix
-    builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
-    ```
-    
-    produces the list `[ 42 77 147 249 483 526 ]`.
-    
-    This is a stable sort: it preserves the relative order of elements
-    deemed equal by the comparator.
-
-  - `builtins.split` *regex* *str*  
-    Returns a list composed of non matched strings interleaved with the
-    lists of the [extended POSIX regular
-    expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
-    *regex* matches of *str*. Each item in the lists of matched
-    sequences is a regex group.
-    
-    ```nix
-    builtins.split "(a)b" "abc"
-    ```
-    
-    Evaluates to `[ "" [ "a" ] "c" ]`.
-    
-    ```nix
-    builtins.split "([ac])" "abc"
-    ```
-    
-    Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`.
-    
-    ```nix
-    builtins.split "(a)|(c)" "abc"
-    ```
-    
-    Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`.
-    
-    ```nix
-    builtins.split "([[:upper:]]+)" "  FOO   "
-    ```
-    
-    Evaluates to `[ " " [ "FOO" ] " " ]`.
-
-  - `builtins.splitVersion` *s*  
-    Split a string representing a version into its components, by the
-    same version splitting logic underlying the version comparison in
-    [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade).
-
-  - `builtins.stringLength` *e*  
-    Return the length of the string *e*. If *e* is not a string,
-    evaluation is aborted.
-
-  - `builtins.sub` *e1* *e2*  
-    Return the difference between the numbers *e1* and *e2*.
-
-  - `builtins.substring` *start* *len* *s*  
-    Return the substring of *s* from character position *start*
-    (zero-based) up to but not including *start + len*. If *start* is
-    greater than the length of the string, an empty string is returned,
-    and if *start + len* lies beyond the end of the string, only the
-    substring up to the end of the string is returned. *start* must be
-    non-negative. For example,
-    
-    ```nix
-    builtins.substring 0 3 "nixos"
-    ```
-    
-    evaluates to `"nix"`.
-
-  - `builtins.tail` *list*  
-    Return the second to last elements of a list; abort evaluation if
-    the argument isn’t a list or is an empty list.
-
-  - `throw` *s*; `builtins.throw` *s*  
-    Throw an error message *s*. This usually aborts Nix expression
-    evaluation, but in `nix-env -qa` and other commands that try to
-    evaluate a set of derivations to get information about those
-    derivations, a derivation that throws an error is silently skipped
-    (which is not the case for `abort`).
-
-  - `builtins.toFile` *name* *s*  
-    Store the string *s* in a file in the Nix store and return its
-    path.  The file has suffix *name*. This file can be used as an
-    input to derivations. One application is to write builders
-    “inline”. For instance, the following Nix expression combines the
-    [Nix expression for GNU Hello](expression-syntax.md) and its
-    [build script](build-script.md) into one file:
-
-    ```nix
-    { stdenv, fetchurl, perl }:
-
-    stdenv.mkDerivation {
-      name = "hello-2.1.1";
-
-      builder = builtins.toFile "builder.sh" "
-        source $stdenv/setup
-
-        PATH=$perl/bin:$PATH
-
-        tar xvfz $src
-        cd hello-*
-        ./configure --prefix=$out
-        make
-        make install
-      ";
-
-      src = fetchurl {
-        url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
-        sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
-      };
-      inherit perl;
-    }
-    ```
-    
-    It is even possible for one file to refer to another, e.g.,
-    
-    ```nix
-    builder = let
-      configFile = builtins.toFile "foo.conf" "
-        # This is some dummy configuration file.
-        ...
-      ";
-    in builtins.toFile "builder.sh" "
-      source $stdenv/setup
-      ...
-      cp ${configFile} $out/etc/foo.conf
-    ";
-  ```
-    
-    Note that `${configFile}` is an
-    [antiquotation](language-values.md), so the result of the
-    expression `configFile`
-    (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
-    spliced into the resulting string.
-    
-    It is however *not* allowed to have files mutually referring to each
-    other, like so:
-    
-    ```nix
-    let
-      foo = builtins.toFile "foo" "...${bar}...";
-      bar = builtins.toFile "bar" "...${foo}...";
-    in foo
-    ```
-    
-    This is not allowed because it would cause a cyclic dependency in
-    the computation of the cryptographic hashes for `foo` and `bar`.
-    
-    It is also not possible to reference the result of a derivation. If
-    you are using Nixpkgs, the `writeTextFile` function is able to do
-    that.
-
-  - `builtins.toJSON` *e*  
-    Return a string containing a JSON representation of *e*. Strings,
-    integers, floats, booleans, nulls and lists are mapped to their JSON
-    equivalents. Sets (except derivations) are represented as objects.
-    Derivations are translated to a JSON string containing the
-    derivation’s output path. Paths are copied to the store and
-    represented as a JSON string of the resulting store path.
-
-  - `builtins.toPath` *s*  
-    DEPRECATED. Use `/. + "/path"` to convert a string into an absolute
-    path. For relative paths, use `./. + "/path"`.
-
-  - `toString` *e*; `builtins.toString` *e*  
-    Convert the expression *e* to a string. *e* can be:
-    
-      - A string (in which case the string is returned unmodified).
-    
-      - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`.
-    
-      - A set containing `{ __toString = self: ...; }`.
-    
-      - An integer.
-    
-      - A list, in which case the string representations of its elements
-        are joined with spaces.
-    
-      - A Boolean (`false` yields `""`, `true` yields `"1"`).
-    
-      - `null`, which yields the empty string.
-
-  - `builtins.toXML` *e*  
-    Return a string containing an XML representation of *e*. The main
-    application for `toXML` is to communicate information with the
-    builder in a more structured format than plain environment
-    variables.
-    
-    Here is an example where this is the case:
-    
-    ```nix
-    { stdenv, fetchurl, libxslt, jira, uberwiki }:
-
-    stdenv.mkDerivation (rec {
-      name = "web-server";
-
-      buildInputs = [ libxslt ];
-
-      builder = builtins.toFile "builder.sh" "
-        source $stdenv/setup
-        mkdir $out
-        echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ① 
-      ";
-
-      stylesheet = builtins.toFile "stylesheet.xsl" ② 
-       "<?xml version='1.0' encoding='UTF-8'?>
-        <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
-          <xsl:template match='/'>
-            <Configure>
-              <xsl:for-each select='/expr/list/attrs'>
-                <Call name='addWebApplication'>
-                  <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg>
-                  <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg>
-                </Call>
-              </xsl:for-each>
-            </Configure>
-          </xsl:template>
-        </xsl:stylesheet>
-      ";
-
-      servlets = builtins.toXML [ ③ 
-        { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
-        { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
-      ];
-    })
-    ```
-    
-    The builder is supposed to generate the configuration file for a
-    [Jetty servlet container](http://jetty.mortbay.org/). A servlet
-    container contains a number of servlets (`*.war` files) each
-    exported under a specific URI prefix. So the servlet configuration
-    is a list of sets containing the `path` and `war` of the servlet
-    (①). This kind of information is difficult to communicate with the
-    normal method of passing information through an environment
-    variable, which just concatenates everything together into a
-    string (which might just work in this case, but wouldn’t work if
-    fields are optional or contain lists themselves). Instead the Nix
-    expression is converted to an XML representation with `toXML`,
-    which is unambiguous and can easily be processed with the
-    appropriate tools. For instance, in the example an XSLT stylesheet
-    (at point ②) is applied to it (at point ①) to generate the XML
-    configuration file for the Jetty server. The XML representation
-    produced at point ③ by `toXML` is as follows:
-    
-    ```xml
-    <?xml version='1.0' encoding='utf-8'?>
-    <expr>
-      <list>
-        <attrs>
-          <attr name="path">
-            <string value="/bugtracker" />
-          </attr>
-          <attr name="war">
-            <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" />
-          </attr>
-        </attrs>
-        <attrs>
-          <attr name="path">
-            <string value="/wiki" />
-          </attr>
-          <attr name="war">
-            <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" />
-          </attr>
-        </attrs>
-      </list>
-    </expr>
-    ```
-    
-    Note that we used the `toFile` built-in to write the builder and
-    the stylesheet “inline” in the Nix expression. The path of the
-    stylesheet is spliced into the builder using the syntax `xsltproc
-    ${stylesheet}`.
-
-  - `builtins.trace` *e1* *e2*  
-    Evaluate *e1* and print its abstract syntax representation on
-    standard error. Then return *e2*. This function is useful for
-    debugging.
-
-  - `builtins.tryEval` *e*  
-    Try to shallowly evaluate *e*. Return a set containing the
-    attributes `success` (`true` if *e* evaluated successfully,
-    `false` if an error was thrown) and `value`, equalling *e* if
-    successful and `false` otherwise. Note that this doesn't evaluate
-    *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval
-    e).success ` will be `true`. Using ` builtins.deepSeq ` one can
-    get the expected result: `let e = { x = throw ""; }; in
-    (builtins.tryEval (builtins.deepSeq e e)).success` will be
-    `false`.
-
-  - `builtins.typeOf` *e*  
-    Return a string representing the type of the value *e*, namely
-    `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`,
-    `"list"`, `"lambda"` or `"float"`.
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 902a37e6b..78892053a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -275,6 +275,16 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
     mkString(v, state.symbols.create(t));
 }
 
+static RegisterPrimOp primop_typeOf({
+    .name = "__typeOf",
+    .args = {"e"},
+    .doc = R"(
+      Return a string representing the type of the value *e*, namely
+      `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`,
+      `"list"`, `"lambda"` or `"float"`.
+    )",
+    .fun = prim_typeOf,
+});
 
 /* Determine whether the argument is the null value. */
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -283,6 +293,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tNull);
 }
 
+static RegisterPrimOp primop_isNull({
+    .name = "isNull",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to `null`, and `false` otherwise.
+
+      > **Warning**
+      > 
+      > This function is *deprecated*; just write `e == null` instead.
+    )",
+    .fun = prim_isNull,
+});
 
 /* Determine whether the argument is a function. */
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -302,6 +324,14 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args,
     mkBool(v, res);
 }
 
+static RegisterPrimOp primop_isFunction({
+    .name = "__isFunction",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a function, and `false` otherwise.
+    )",
+    .fun = prim_isFunction,
+});
 
 /* Determine whether the argument is an integer. */
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -310,6 +340,15 @@ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value
     mkBool(v, args[0]->type == tInt);
 }
 
+static RegisterPrimOp primop_isInt({
+    .name = "__isInt",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to an integer, and `false` otherwise.
+    )",
+    .fun = prim_isInt,
+});
+
 /* Determine whether the argument is a float. */
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -317,6 +356,15 @@ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Val
     mkBool(v, args[0]->type == tFloat);
 }
 
+static RegisterPrimOp primop_isFloat({
+    .name = "__isFloat",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a float, and `false` otherwise.
+    )",
+    .fun = prim_isFloat,
+});
+
 /* Determine whether the argument is a string. */
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -324,6 +372,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va
     mkBool(v, args[0]->type == tString);
 }
 
+static RegisterPrimOp primop_isString({
+    .name = "__isString",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a string, and `false` otherwise.
+    )",
+    .fun = prim_isString,
+});
 
 /* Determine whether the argument is a Boolean. */
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -332,6 +388,15 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tBool);
 }
 
+static RegisterPrimOp primop_isBool({
+    .name = "__isBool",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a bool, and `false` otherwise.
+    )",
+    .fun = prim_isBool,
+});
+
 /* Determine whether the argument is a path. */
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -339,6 +404,15 @@ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tPath);
 }
 
+static RegisterPrimOp primop_isPath({
+    .name = "__isPath",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a path, and `false` otherwise.
+    )",
+    .fun = prim_isPath,
+});
+
 struct CompareValues
 {
     bool operator () (const Value * v1, const Value * v2) const
@@ -459,14 +533,23 @@ static RegisterPrimOp primop_abort({
     }
 });
 
-
-static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    throw ThrownError(s);
-}
-
+static RegisterPrimOp primop_throw({
+    .name = "throw",
+    .args = {"s"},
+    .doc = R"(
+      Throw an error message *s*. This usually aborts Nix expression
+      evaluation, but in `nix-env -qa` and other commands that try to
+      evaluate a set of derivations to get information about those
+      derivations, a derivation that throws an error is silently skipped
+      (which is not the case for `abort`).
+    )",
+    .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v)
+    {
+      PathSet context;
+      string s = state.coerceToString(pos, *args[0], context);
+      throw ThrownError(s);
+    }
+});
 
 static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -497,6 +580,22 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_tryEval({
+    .name = "__tryEval",
+    .args = {"e"},
+    .doc = R"(
+      Try to shallowly evaluate *e*. Return a set containing the
+      attributes `success` (`true` if *e* evaluated successfully,
+      `false` if an error was thrown) and `value`, equalling *e* if
+      successful and `false` otherwise. Note that this doesn't evaluate
+      *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval
+      e).success ` will be `true`. Using ` builtins.deepSeq ` one can
+      get the expected result: `let e = { x = throw ""; }; in
+      (builtins.tryEval (builtins.deepSeq e e)).success` will be
+      `false`.
+    )",
+    .fun = prim_tryEval,
+});
 
 /* Return an environment variable.  Use with care. */
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -505,6 +604,22 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu
     mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
+static RegisterPrimOp primop_getEnv({
+    .name = "__getEnv",
+    .args = {"s"},
+    .doc = R"(
+      `getEnv` returns the value of the environment variable *s*, or an
+      empty string if the variable doesn’t exist. This function should be
+      used with care, as it can introduce all sorts of nasty environment
+      dependencies in your Nix expression.
+
+      `getEnv` is used in Nix Packages to locate the file
+      `~/.nixpkgs/config.nix`, which contains user-local settings for Nix
+      Packages. (That is, it does a `getEnv "HOME"` to locate the user’s
+      home directory.)
+    )",
+    .fun = prim_getEnv,
+});
 
 /* Evaluate the first argument, then return the second argument. */
 static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -514,6 +629,15 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value &
     v = *args[1];
 }
 
+static RegisterPrimOp primop_seq({
+    .name = "__seq",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Evaluate *e1*, then evaluate and return *e2*. This ensures that a
+      computation is strict in the value of *e1*.
+    )",
+    .fun = prim_seq,
+});
 
 /* Evaluate the first argument deeply (i.e. recursing into lists and
    attrsets), then return the second argument. */
@@ -524,6 +648,16 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val
     v = *args[1];
 }
 
+static RegisterPrimOp primop_deepSeq({
+    .name = "__deepSeq",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      This is like `seq e1 e2`, except that *e1* is evaluated *deeply*:
+      if it’s a list or set, its elements or attributes are also
+      evaluated recursively.
+    )",
+    .fun = prim_deepSeq,
+});
 
 /* Evaluate the first expression and print it on standard error.  Then
    return the second expression.  Useful for debugging. */
@@ -538,6 +672,17 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value
     v = *args[1];
 }
 
+static RegisterPrimOp primop_trace({
+    .name = "__trace",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Evaluate *e1* and print its abstract syntax representation on
+      standard error. Then return *e2*. This function is useful for
+      debugging.
+    )",
+    .fun = prim_trace,
+});
+
 
 /*************************************************************
  * Derivations
@@ -879,6 +1024,17 @@ static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args,
     mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos)));
 }
 
+static RegisterPrimOp primop_placeholder({
+    .name = "placeholder",
+    .args = {"output"},
+    .doc = R"(
+      Return a placeholder string for the specified *output* that will be
+      substituted by the corresponding output path at build time. Typical
+      outputs would be `"out"`, `"bin"` or `"dev"`.
+    )",
+    .fun = prim_placeholder,
+});
+
 
 /*************************************************************
  * Paths
@@ -893,6 +1049,15 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu
     mkString(v, canonPath(path), context);
 }
 
+static RegisterPrimOp primop_toPath({
+    .name = "__toPath",
+    .args = {"s"},
+    .doc = R"(
+      DEPRECATED. Use `/. + "/path"` to convert a string into an absolute
+      path. For relative paths, use `./. + "/path"`.
+    )",
+    .fun = prim_toPath,
+});
 
 /* Allow a valid store path to be used in an expression.  This is
    useful in some generated expressions such as in nix-push, which
@@ -904,6 +1069,9 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu
    corner cases. */
 static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    if (evalSettings.pureEval)
+        throw EvalError("builtins.storePath' is not allowed in pure evaluation mode");
+
     PathSet context;
     Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context));
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
@@ -949,6 +1117,15 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     }
 }
 
+static RegisterPrimOp primop_pathExists({
+    .name = "__pathExists",
+    .args = {"path"},
+    .doc = R"(
+      Return `true` if the path *path* exists at evaluation time, and
+      `false` otherwise.
+    )",
+    .fun = prim_pathExists,
+});
 
 /* Return the base name of the given string, i.e., everything
    following the last slash. */
@@ -958,6 +1135,16 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args,
     mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context);
 }
 
+static RegisterPrimOp primop_baseNameOf({
+    .name = "baseNameOf",
+    .args = {"s"},
+    .doc = R"(
+      Return the *base name* of the string *s*, that is, everything
+      following the final slash in the string. This is similar to the GNU
+      `basename` command.
+    )",
+    .fun = prim_baseNameOf,
+});
 
 /* Return the directory of the given path, i.e., everything before the
    last slash.  Return either a path or a string depending on the type
@@ -969,6 +1156,16 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value
     if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
+static RegisterPrimOp primop_dirOf({
+    .name = "dirOf",
+    .args = {"s"},
+    .doc = R"(
+      Return the directory part of the string *s*, that is, everything
+      before the final slash in the string. This is similar to the GNU
+      `dirname` command.
+    )",
+    .fun = prim_dirOf,
+});
 
 /* Return the contents of a file as a string. */
 static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -989,6 +1186,14 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkString(v, s.c_str());
 }
 
+static RegisterPrimOp primop_readFile({
+    .name = "__readFile",
+    .args = {"path"},
+    .doc = R"(
+      Return the contents of the file *path* as a string.
+    )",
+    .fun = prim_readFile,
+});
 
 /* Find a file in the Nix search path. Used to implement <x> paths,
    which are desugared to 'findFile __nixPath "x"'. */
@@ -1051,6 +1256,17 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
 }
 
+static RegisterPrimOp primop_hashFile({
+    .name = "__hashFile",
+    .args = {"type", "p"},
+    .doc = R"(
+      Return a base-16 representation of the cryptographic hash of the
+      file at path *p*. The hash algorithm specified by *type* must be one
+      of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`.
+    )",
+    .fun = prim_hashFile,
+});
+
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1082,6 +1298,25 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_readDir({
+    .name = "__readDir",
+    .args = {"path"},
+    .doc = R"(
+      Return the contents of the directory *path* as a set mapping
+      directory entries to the corresponding file type. For instance, if
+      directory `A` contains a regular file `B` and another directory
+      `C`, then `builtins.readDir ./A` will return the set
+
+      ```nix
+      { B = "regular"; C = "directory"; }
+      ```
+
+      The possible values for the file type are `"regular"`,
+      `"directory"`, `"symlink"` and `"unknown"`.
+    )",
+    .fun = prim_readDir,
+});
+
 
 /*************************************************************
  * Creating files
@@ -1099,6 +1334,102 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value
     mkString(v, out.str(), context);
 }
 
+static RegisterPrimOp primop_toXML({
+    .name = "__toXML",
+    .args = {"e"},
+    .doc = R"(
+      Return a string containing an XML representation of *e*. The main
+      application for `toXML` is to communicate information with the
+      builder in a more structured format than plain environment
+      variables.
+
+      Here is an example where this is the case:
+
+      ```nix
+      { stdenv, fetchurl, libxslt, jira, uberwiki }:
+
+      stdenv.mkDerivation (rec {
+        name = "web-server";
+
+        buildInputs = [ libxslt ];
+
+        builder = builtins.toFile "builder.sh" "
+          source $stdenv/setup
+          mkdir $out
+          echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ①
+        ";
+
+        stylesheet = builtins.toFile "stylesheet.xsl" ②
+         "<?xml version='1.0' encoding='UTF-8'?>
+          <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
+            <xsl:template match='/'>
+              <Configure>
+                <xsl:for-each select='/expr/list/attrs'>
+                  <Call name='addWebApplication'>
+                    <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg>
+                    <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg>
+                  </Call>
+                </xsl:for-each>
+              </Configure>
+            </xsl:template>
+          </xsl:stylesheet>
+        ";
+
+        servlets = builtins.toXML [ ③
+          { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; }
+          { path = "/wiki"; war = uberwiki + "/uberwiki.war"; }
+        ];
+      })
+      ```
+
+      The builder is supposed to generate the configuration file for a
+      [Jetty servlet container](http://jetty.mortbay.org/). A servlet
+      container contains a number of servlets (`*.war` files) each
+      exported under a specific URI prefix. So the servlet configuration
+      is a list of sets containing the `path` and `war` of the servlet
+      (①). This kind of information is difficult to communicate with the
+      normal method of passing information through an environment
+      variable, which just concatenates everything together into a
+      string (which might just work in this case, but wouldn’t work if
+      fields are optional or contain lists themselves). Instead the Nix
+      expression is converted to an XML representation with `toXML`,
+      which is unambiguous and can easily be processed with the
+      appropriate tools. For instance, in the example an XSLT stylesheet
+      (at point ②) is applied to it (at point ①) to generate the XML
+      configuration file for the Jetty server. The XML representation
+      produced at point ③ by `toXML` is as follows:
+
+      ```xml
+      <?xml version='1.0' encoding='utf-8'?>
+      <expr>
+        <list>
+          <attrs>
+            <attr name="path">
+              <string value="/bugtracker" />
+            </attr>
+            <attr name="war">
+              <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" />
+            </attr>
+          </attrs>
+          <attrs>
+            <attr name="path">
+              <string value="/wiki" />
+            </attr>
+            <attr name="war">
+              <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" />
+            </attr>
+          </attrs>
+        </list>
+      </expr>
+      ```
+
+      Note that we used the `toFile` built-in to write the builder and
+      the stylesheet “inline” in the Nix expression. The path of the
+      stylesheet is spliced into the builder using the syntax `xsltproc
+      ${stylesheet}`.
+    )",
+    .fun = prim_toXML,
+});
 
 /* Convert the argument (which can be any Nix expression) to a JSON
    string.  Not all Nix expressions can be sensibly or completely
@@ -1111,6 +1442,19 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu
     mkString(v, out.str(), context);
 }
 
+static RegisterPrimOp primop_toJSON({
+    .name = "__toJSON",
+    .args = {"e"},
+    .doc = R"(
+      Return a string containing a JSON representation of *e*. Strings,
+      integers, floats, booleans, nulls and lists are mapped to their JSON
+      equivalents. Sets (except derivations) are represented as objects.
+      Derivations are translated to a JSON string containing the
+      derivation’s output path. Paths are copied to the store and
+      represented as a JSON string of the resulting store path.
+    )",
+    .fun = prim_toJSON,
+});
 
 /* Parse a JSON string to a value. */
 static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1119,6 +1463,20 @@ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Va
     parseJSON(state, s, v);
 }
 
+static RegisterPrimOp primop_fromJSON({
+    .name = "__fromJSON",
+    .args = {"e"},
+    .doc = R"(
+      Convert a JSON string to a Nix value. For example,
+
+      ```nix
+      builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
+      ```
+
+      returns the value `{ x = [ 1 2 3 ]; y = null; }`.
+    )",
+    .fun = prim_fromJSON,
+});
 
 /* Store a string in the Nix store as a source file that can be used
    as an input by derivations. */
@@ -1153,6 +1511,83 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     mkString(v, storePath, {storePath});
 }
 
+static RegisterPrimOp primop_toFile({
+    .name = "__toFile",
+    .args = {"name", "s"},
+    .doc = R"(
+      Store the string *s* in a file in the Nix store and return its
+      path.  The file has suffix *name*. This file can be used as an
+      input to derivations. One application is to write builders
+      “inline”. For instance, the following Nix expression combines the
+      [Nix expression for GNU Hello](expression-syntax.md) and its
+      [build script](build-script.md) into one file:
+
+      ```nix
+      { stdenv, fetchurl, perl }:
+
+      stdenv.mkDerivation {
+        name = "hello-2.1.1";
+
+        builder = builtins.toFile "builder.sh" "
+          source $stdenv/setup
+
+          PATH=$perl/bin:$PATH
+
+          tar xvfz $src
+          cd hello-*
+          ./configure --prefix=$out
+          make
+          make install
+        ";
+
+        src = fetchurl {
+          url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
+          sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
+        };
+        inherit perl;
+      }
+      ```
+
+      It is even possible for one file to refer to another, e.g.,
+
+      ```nix
+      builder = let
+        configFile = builtins.toFile "foo.conf" "
+          # This is some dummy configuration file.
+          ...
+        ";
+      in builtins.toFile "builder.sh" "
+        source $stdenv/setup
+        ...
+        cp ${configFile} $out/etc/foo.conf
+      ";
+    ```
+
+      Note that `${configFile}` is an
+      [antiquotation](language-values.md), so the result of the
+      expression `configFile`
+      (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
+      spliced into the resulting string.
+
+      It is however *not* allowed to have files mutually referring to each
+      other, like so:
+
+      ```nix
+      let
+        foo = builtins.toFile "foo" "...${bar}...";
+        bar = builtins.toFile "bar" "...${foo}...";
+      in foo
+      ```
+
+      This is not allowed because it would cause a cyclic dependency in
+      the computation of the cryptographic hashes for `foo` and `bar`.
+
+      It is also not possible to reference the result of a derivation. If
+      you are using Nixpkgs, the `writeTextFile` function is able to do
+      that.
+    )",
+    .fun = prim_toFile,
+});
 
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
     Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
@@ -1223,6 +1658,48 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
 }
 
+static RegisterPrimOp primop_filterSource({
+    .name = "__filterSource",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      This function allows you to copy sources into the Nix store while
+      filtering certain files. For instance, suppose that you want to use
+      the directory `source-dir` as an input to a Nix expression, e.g.
+
+      ```nix
+      stdenv.mkDerivation {
+        ...
+        src = ./source-dir;
+      }
+      ```
+
+      However, if `source-dir` is a Subversion working copy, then all
+      those annoying `.svn` subdirectories will also be copied to the
+      store. Worse, the contents of those directories may change a lot,
+      causing lots of spurious rebuilds. With `filterSource` you can
+      filter out the `.svn` directories:
+
+      ```nix
+      src = builtins.filterSource
+        (path: type: type != "directory" || baseNameOf path != ".svn")
+        ./source-dir;
+      ```
+
+      Thus, the first argument *e1* must be a predicate function that is
+      called for each regular file, directory or symlink in the source
+      tree *e2*. If the function returns `true`, the file is copied to the
+      Nix store, otherwise it is omitted. The function is called with two
+      arguments. The first is the full path of the file. The second is a
+      string that identifies the type of the file, which is either
+      `"regular"`, `"directory"`, `"symlink"` or `"unknown"` (for other
+      kinds of files such as device nodes or fifos — but note that those
+      cannot be copied to the Nix store, so if the predicate returns
+      `true` for them, the copy will fail). If you exclude a directory,
+      the entire corresponding subtree of *e2* will be excluded.
+    )",
+    .fun = prim_filterSource,
+});
+
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], pos);
@@ -1268,6 +1745,41 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     addPath(state, pos, name, path, filterFun, method, expectedHash, v);
 }
 
+static RegisterPrimOp primop_path({
+    .name = "__path",
+    .args = {"args"},
+    .doc = R"(
+      An enrichment of the built-in path type, based on the attributes
+      present in *args*. All are optional except `path`:
+
+        - path  
+          The underlying path.
+
+        - name  
+          The name of the path when added to the store. This can used to
+          reference paths that have nix-illegal characters in their names,
+          like `@`.
+
+        - filter  
+          A function of the type expected by `builtins.filterSource`,
+          with the same semantics.
+
+        - recursive  
+          When `false`, when `path` is added to the store it is with a
+          flat hash, rather than a hash of the NAR serialization of the
+          file. Thus, `path` must refer to a regular file, not a
+          directory. This allows similar behavior to `fetchurl`. Defaults
+          to `true`.
+
+        - sha256  
+          When provided, this is the expected hash of the file at the
+          path. Evaluation will fail if the hash is incorrect, and
+          providing a hash allows `builtins.path` to be used even when the
+          `pure-eval` nix config option is on.
+    )",
+    .fun = prim_path,
+});
+
 
 /*************************************************************
  * Sets
@@ -1290,6 +1802,16 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
               [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
 }
 
+static RegisterPrimOp primop_attrNames({
+    .name = "__attrNames",
+    .args = {"set"},
+    .doc = R"(
+      Return the names of the attributes in the set *set* in an
+      alphabetically sorted list. For instance, `builtins.attrNames { y
+      = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`.
+    )",
+    .fun = prim_attrNames,
+});
 
 /* Return the values of the attributes in a set as a list, in the same
    order as attrNames. */
@@ -1310,6 +1832,15 @@ static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args,
         v.listElems()[i] = ((Attr *) v.listElems()[i])->value;
 }
 
+static RegisterPrimOp primop_attrValues({
+    .name = "__attrValues",
+    .args = {"set"},
+    .doc = R"(
+      Return the values of the attributes in the set *set* in the order
+      corresponding to the sorted attribute names.
+    )",
+    .fun = prim_attrValues,
+});
 
 /* Dynamic version of the `.' operator. */
 void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1329,9 +1860,20 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     v = *i->value;
 }
 
+static RegisterPrimOp primop_getAttr({
+    .name = "__getAttr",
+    .args = {"s", "set"},
+    .doc = R"(
+      `getAttr` returns the attribute named *s* from *set*. Evaluation
+      aborts if the attribute doesn’t exist. This is a dynamic version of
+      the `.` operator, since *s* is an expression rather than an
+      identifier.
+    )",
+    .fun = prim_getAttr,
+});
 
 /* Return position information of the specified attribute. */
-void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string attr = state.forceStringNoCtx(*args[0], pos);
     state.forceAttrs(*args[1], pos);
@@ -1342,7 +1884,6 @@ void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, V
         state.mkPos(v, i->pos);
 }
 
-
 /* Dynamic version of the `?' operator. */
 static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1351,6 +1892,16 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val
     mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end());
 }
 
+static RegisterPrimOp primop_hasAttr({
+    .name = "__hasAttr",
+    .args = {"s", "set"},
+    .doc = R"(
+      `hasAttr` returns `true` if *set* has an attribute named *s*, and
+      `false` otherwise. This is a dynamic version of the `?` operator,
+      since *s* is an expression rather than an identifier.
+    )",
+    .fun = prim_hasAttr,
+});
 
 /* Determine whether the argument is a set. */
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1359,6 +1910,14 @@ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Val
     mkBool(v, args[0]->type == tAttrs);
 }
 
+static RegisterPrimOp primop_isAttrs({
+    .name = "__isAttrs",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a set, and `false` otherwise.
+    )",
+    .fun = prim_isAttrs,
+});
 
 static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1382,6 +1941,21 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
     }
 }
 
+static RegisterPrimOp primop_removeAttrs({
+    .name = "removeAttrs",
+    .args = {"set", "list"},
+    .doc = R"(
+      Remove the attributes listed in *list* from *set*. The attributes
+      don’t have to exist in *set*. For instance,
+
+      ```nix
+      removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]
+      ```
+
+      evaluates to `{ y = 2; }`.
+    )",
+    .fun = prim_removeAttrs,
+});
 
 /* Builds a set from a list specifying (name, value) pairs.  To be
    precise, a list [{name = "name1"; value = value1;} ... {name =
@@ -1423,6 +1997,30 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_listToAttrs({
+    .name = "__listToAttrs",
+    .args = {"e"},
+    .doc = R"(
+      Construct a set from a list specifying the names and values of each
+      attribute. Each element of the list should be a set consisting of a
+      string-valued attribute `name` specifying the name of the attribute,
+      and an attribute `value` specifying its value. Example:
+
+      ```nix
+      builtins.listToAttrs
+        [ { name = "foo"; value = 123; }
+          { name = "bar"; value = 456; }
+        ]
+      ```
+
+      evaluates to
+
+      ```nix
+      { foo = 123; bar = 456; }
+      ```
+    )",
+    .fun = prim_listToAttrs,
+});
 
 /* Return the right-biased intersection of two sets as1 and as2,
    i.e. a set that contains every attribute from as2 that is also a
@@ -1441,6 +2039,15 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar
     }
 }
 
+static RegisterPrimOp primop_intersectAttrs({
+    .name = "__intersectAttrs",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return a set consisting of the attributes in the set *e2* that also
+      exist in the set *e1*.
+    )",
+    .fun = prim_intersectAttrs,
+});
 
 /* Collect each attribute named `attr' from a list of attribute sets.
    Sets that don't contain the named attribute are ignored.
@@ -1470,7 +2077,6 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
         v.listElems()[n] = res[n];
 }
 
-
 /* Return a set containing the names of the formal arguments expected
    by the function `f'.  The value of each attribute is a Boolean
    denoting whether the corresponding argument has a default value.  For instance,
@@ -1508,6 +2114,22 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_functionArgs({
+    .name = "__functionArgs",
+    .args = {"f"},
+    .doc = R"(
+      Return a set containing the names of the formal arguments expected
+      by the function *f*. The value of each attribute is a Boolean
+      denoting whether the corresponding argument has a default value. For
+      instance, `functionArgs ({ x, y ? 123}: ...) = { x = false; y =
+      true; }`.
+
+      "Formal argument" here refers to the attributes pattern-matched by
+      the function. Plain lambdas are not included, e.g. `functionArgs (x:
+      ...) = { }`.
+    )",
+    .fun = prim_functionArgs,
+});
 
 /* Apply a function to every element of an attribute set. */
 static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1526,7 +2148,6 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 }
 
 
-
 /*************************************************************
  * Lists
  *************************************************************/
@@ -1539,6 +2160,14 @@ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->isList());
 }
 
+static RegisterPrimOp primop_isList({
+    .name = "__isList",
+    .args = {"e"},
+    .doc = R"(
+      Return `true` if *e* evaluates to a list, and `false` otherwise.
+    )",
+    .fun = prim_isList,
+});
 
 static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v)
 {
@@ -1552,13 +2181,21 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     v = *list.listElems()[n];
 }
 
-
 /* Return the n-1'th element of a list. */
 static void prim_elemAt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v);
 }
 
+static RegisterPrimOp primop_elemAt({
+    .name = "__elemAt",
+    .args = {"xs", "n"},
+    .doc = R"(
+      Return element *n* from the list *xs*. Elements are counted starting
+      from 0. A fatal error occurs if the index is out of bounds.
+    )",
+    .fun = prim_elemAt,
+});
 
 /* Return the first element of a list. */
 static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1566,6 +2203,16 @@ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value
     elemAt(state, pos, *args[0], 0, v);
 }
 
+static RegisterPrimOp primop_head({
+    .name = "__head",
+    .args = {"list"},
+    .doc = R"(
+      Return the first element of a list; abort evaluation if the argument
+      isn’t a list or is an empty list. You can test whether a list is
+      empty by comparing it with `[]`.
+    )",
+    .fun = prim_head,
+});
 
 /* Return a list consisting of everything but the first element of
    a list.  Warning: this function takes O(n) time, so you probably
@@ -1584,6 +2231,21 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
         v.listElems()[n] = args[0]->listElems()[n + 1];
 }
 
+static RegisterPrimOp primop_tail({
+    .name = "__tail",
+    .args = {"list"},
+    .doc = R"(
+      Return the second to last elements of a list; abort evaluation if
+      the argument isn’t a list or is an empty list.
+
+      > **Warning**
+      > 
+      > This function should generally be avoided since it's inefficient:
+      > unlike Haskell's `tail`, it takes O(n) time, so recursing over a
+      > list by repeatedly calling `tail` takes O(n^2) time.
+    )",
+    .fun = prim_tail,
+});
 
 /* Apply a function to every element of a list. */
 static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1597,6 +2259,21 @@ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value &
             *args[0], *args[1]->listElems()[n]);
 }
 
+static RegisterPrimOp primop_map({
+    .name = "map",
+    .args = {"f", "list"},
+    .doc = R"(
+      Apply the function *f* to each element in the list *list*. For
+      example,
+
+      ```nix
+      map (x: "foo" + x) [ "bar" "bla" "abc" ]
+      ```
+
+      evaluates to `[ "foobar" "foobla" "fooabc" ]`.
+    )",
+    .fun = prim_map,
+});
 
 /* Filter a list using a predicate; that is, return a list containing
    every element from the list for which the predicate function
@@ -1628,6 +2305,15 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu
     }
 }
 
+static RegisterPrimOp primop_filter({
+    .name = "__filter",
+    .args = {"f", "list"},
+    .doc = R"(
+      Return a list consisting of the elements of *list* for which the
+      function *f* returns `true`.
+    )",
+    .fun = prim_filter,
+});
 
 /* Return true if a list contains a given element. */
 static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1642,6 +2328,15 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value
     mkBool(v, res);
 }
 
+static RegisterPrimOp primop_elem({
+    .name = "__elem",
+    .args = {"x", "xs"},
+    .doc = R"(
+      Return `true` if a value equal to *x* occurs in the list *xs*, and
+      `false` otherwise.
+    )",
+    .fun = prim_elem,
+});
 
 /* Concatenate a list of lists. */
 static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1650,6 +2345,14 @@ static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args,
     state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos);
 }
 
+static RegisterPrimOp primop_concatLists({
+    .name = "__concatLists",
+    .args = {"lists"},
+    .doc = R"(
+      Concatenate a list of lists into a single list.
+    )",
+    .fun = prim_concatLists,
+});
 
 /* Return the length of a list.  This is an O(1) time operation. */
 static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1658,6 +2361,14 @@ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Valu
     mkInt(v, args[0]->listSize());
 }
 
+static RegisterPrimOp primop_length({
+    .name = "__length",
+    .args = {"e"},
+    .doc = R"(
+      Return the length of the list *e*.
+    )",
+    .fun = prim_length,
+});
 
 /* Reduce a list by applying a binary operator, from left to
    right. The operator is applied strictly. */
@@ -1682,6 +2393,18 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     }
 }
 
+static RegisterPrimOp primop_foldlStrict({
+    .name = "__foldl'",
+    .args = {"op", "nul", "list"},
+    .doc = R"(
+      Reduce a list by applying a binary operator, from left to right,
+      e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
+      ...`. The operator is applied strictly, i.e., its arguments are
+      evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]`
+      evaluates to 6.
+    )",
+    .fun = prim_foldlStrict,
+});
 
 static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1707,12 +2430,30 @@ static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value &
     anyOrAll(true, state, pos, args, v);
 }
 
+static RegisterPrimOp primop_any({
+    .name = "__any",
+    .args = {"pred", "list"},
+    .doc = R"(
+      Return `true` if the function *pred* returns `true` for at least one
+      element of *list*, and `false` otherwise.
+    )",
+    .fun = prim_any,
+});
 
 static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     anyOrAll(false, state, pos, args, v);
 }
 
+static RegisterPrimOp primop_all({
+    .name = "__all",
+    .args = {"pred", "list"},
+    .doc = R"(
+      Return `true` if the function *pred* returns `true` for all elements
+      of *list*, and `false` otherwise.
+    )",
+    .fun = prim_all,
+});
 
 static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1733,6 +2474,21 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     }
 }
 
+static RegisterPrimOp primop_genList({
+    .name = "__genList",
+    .args = {"generator", "length"},
+    .doc = R"(
+      Generate list of size *length*, with each element *i* equal to the
+      value returned by *generator* `i`. For example,
+
+      ```nix
+      builtins.genList (x: x * x) 5
+      ```
+
+      returns the list `[ 0 1 4 9 16 ]`.
+    )",
+    .fun = prim_genList,
+});
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v);
 
@@ -1768,6 +2524,26 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
     std::stable_sort(v.listElems(), v.listElems() + len, comparator);
 }
 
+static RegisterPrimOp primop_sort({
+    .name = "__sort",
+    .args = {"comparator", "list"},
+    .doc = R"(
+      Return *list* in sorted order. It repeatedly calls the function
+      *comparator* with two elements. The comparator should return `true`
+      if the first element is less than the second, and `false` otherwise.
+      For example,
+
+      ```nix
+      builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
+      ```
+
+      produces the list `[ 42 77 147 249 483 526 ]`.
+
+      This is a stable sort: it preserves the relative order of elements
+      deemed equal by the comparator.
+    )",
+    .fun = prim_sort,
+});
 
 static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1851,6 +2627,14 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
         mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos));
 }
 
+static RegisterPrimOp primop_add({
+    .name = "__add",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the sum of the numbers *e1* and *e2*.
+    )",
+    .fun = prim_add,
+});
 
 static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1862,6 +2646,14 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
         mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos));
 }
 
+static RegisterPrimOp primop_sub({
+    .name = "__sub",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the difference between the numbers *e1* and *e2*.
+    )",
+    .fun = prim_sub,
+});
 
 static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1873,6 +2665,14 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
         mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos));
 }
 
+static RegisterPrimOp primop_mul({
+    .name = "__mul",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the product of the numbers *e1* and *e2*.
+    )",
+    .fun = prim_mul,
+});
 
 static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1902,21 +2702,57 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
+static RegisterPrimOp primop_div({
+    .name = "__div",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the quotient of the numbers *e1* and *e2*.
+    )",
+    .fun = prim_div,
+});
+
 static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
 }
 
+static RegisterPrimOp primop_bitAnd({
+    .name = "__bitAnd",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the bitwise AND of the integers *e1* and *e2*.
+    )",
+    .fun = prim_bitAnd,
+});
+
 static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos));
 }
 
+static RegisterPrimOp primop_bitOr({
+    .name = "__bitOr",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the bitwise OR of the integers *e1* and *e2*.
+    )",
+    .fun = prim_bitOr,
+});
+
 static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
 }
 
+static RegisterPrimOp primop_bitXor({
+    .name = "__bitXor",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return the bitwise XOR of the integers *e1* and *e2*.
+    )",
+    .fun = prim_bitXor,
+});
+
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
@@ -1925,6 +2761,17 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va
     mkBool(v, comp(args[0], args[1]));
 }
 
+static RegisterPrimOp primop_lessThan({
+    .name = "__lessThan",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Return `true` if the number *e1* is less than the number *e2*, and
+      `false` otherwise. Evaluation aborts if either *e1* or *e2* does not
+      evaluate to a number.
+    )",
+    .fun = prim_lessThan,
+});
+
 
 /*************************************************************
  * String manipulation
@@ -1941,6 +2788,29 @@ static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Va
     mkString(v, s, context);
 }
 
+static RegisterPrimOp primop_toString({
+    .name = "toString",
+    .args = {"e"},
+    .doc = R"(
+      Convert the expression *e* to a string. *e* can be:
+
+        - A string (in which case the string is returned unmodified).
+
+        - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`.
+
+        - A set containing `{ __toString = self: ...; }`.
+
+        - An integer.
+
+        - A list, in which case the string representations of its elements
+          are joined with spaces.
+
+        - A Boolean (`false` yields `""`, `true` yields `"1"`).
+
+        - `null`, which yields the empty string.
+    )",
+    .fun = prim_toString,
+});
 
 /* `substring start len str' returns the substring of `str' starting
    at character position `min(start, stringLength str)' inclusive and
@@ -1962,6 +2832,25 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
 
+static RegisterPrimOp primop_substring({
+    .name = "__substring",
+    .args = {"start", "len", "s"},
+    .doc = R"(
+      Return the substring of *s* from character position *start*
+      (zero-based) up to but not including *start + len*. If *start* is
+      greater than the length of the string, an empty string is returned,
+      and if *start + len* lies beyond the end of the string, only the
+      substring up to the end of the string is returned. *start* must be
+      non-negative. For example,
+
+      ```nix
+      builtins.substring 0 3 "nixos"
+      ```
+
+      evaluates to `"nix"`.
+    )",
+    .fun = prim_substring,
+});
 
 static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1970,6 +2859,15 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
     mkInt(v, s.size());
 }
 
+static RegisterPrimOp primop_stringLength({
+    .name = "__stringLength",
+    .args = {"e"},
+    .doc = R"(
+      Return the length of the string *e*. If *e* is not a string,
+      evaluation is aborted.
+    )",
+    .fun = prim_stringLength,
+});
 
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1988,6 +2886,16 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     mkString(v, hashString(*ht, s).to_string(Base16, false), context);
 }
 
+static RegisterPrimOp primop_hashString({
+    .name = "__hashString",
+    .args = {"type", "s"},
+    .doc = R"(
+      Return a base-16 representation of the cryptographic hash of string
+      *s*. The hash algorithm specified by *type* must be one of `"md5"`,
+      `"sha1"`, `"sha256"` or `"sha512"`.
+    )",
+    .fun = prim_hashString,
+});
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
@@ -2036,6 +2944,41 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
     }
 }
 
+static RegisterPrimOp primop_match({
+    .name = "__match",
+    .args = {"regex", "str"},
+    .doc = R"s(
+      Returns a list if the [extended POSIX regular
+      expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
+      *regex* matches *str* precisely, otherwise returns `null`. Each item
+      in the list is a regex group.
+
+      ```nix
+      builtins.match "ab" "abc"
+      ```
+
+      Evaluates to `null`.
+
+      ```nix
+      builtins.match "abc" "abc"
+      ```
+
+      Evaluates to `[ ]`.
+
+      ```nix
+      builtins.match "a(b)(c)" "abc"
+      ```
+
+      Evaluates to `[ "b" "c" ]`.
+
+      ```nix
+      builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
+      ```
+
+      Evaluates to `[ "foo" ]`.
+    )s",
+    .fun = prim_match,
+});
 
 /* Split a string with a regular expression, and return a list of the
    non-matching parts interleaved by the lists of the matching groups. */
@@ -2109,8 +3052,44 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
     }
 }
 
+static RegisterPrimOp primop_split({
+    .name = "__split",
+    .args = {"regex", "str"},
+    .doc = R"s(
+      Returns a list composed of non matched strings interleaved with the
+      lists of the [extended POSIX regular
+      expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)
+      *regex* matches of *str*. Each item in the lists of matched
+      sequences is a regex group.
 
-static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v)
+      ```nix
+      builtins.split "(a)b" "abc"
+      ```
+
+      Evaluates to `[ "" [ "a" ] "c" ]`.
+
+      ```nix
+      builtins.split "([ac])" "abc"
+      ```
+
+      Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`.
+
+      ```nix
+      builtins.split "(a)|(c)" "abc"
+      ```
+
+      Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`.
+
+      ```nix
+      builtins.split "([[:upper:]]+)" "  FOO   "
+      ```
+
+      Evaluates to `[ " " [ "FOO" ] " " ]`.
+    )s",
+    .fun = prim_split,
+});
+
+static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
 
@@ -2129,6 +3108,16 @@ static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * a
     mkString(v, res, context);
 }
 
+static RegisterPrimOp primop_concatStringsSep({
+    .name = "__concatStringsSep",
+    .args = {"separator", "list"},
+    .doc = R"(
+      Concatenate a list of strings with a separator between each
+      element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] ==
+      "usr/local/bin"`.
+    )",
+    .fun = prim_concatStringsSep,
+});
 
 static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2188,6 +3177,22 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     mkString(v, res, context);
 }
 
+static RegisterPrimOp primop_replaceStrings({
+    .name = "__replaceStrings",
+    .args = {"from", "to", "s"},
+    .doc = R"(
+      Given string *s*, replace every occurrence of the strings in *from*
+      with the corresponding string in *to*. For example,
+
+      ```nix
+      builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
+      ```
+
+      evaluates to `"fabir"`.
+    )",
+    .fun = prim_replaceStrings,
+});
+
 
 /*************************************************************
  * Versions
@@ -2204,6 +3209,19 @@ static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_parseDrvName({
+    .name = "__parseDrvName",
+    .args = {"s"},
+    .doc = R"(
+      Split the string *s* into a package name and version. The package
+      name is everything up to but not including the first dash followed
+      by a digit, and the version is everything following that dash. The
+      result is returned in a set `{ name, version }`. Thus,
+      `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name =
+      "nix"; version = "0.12pre12876"; }`.
+    )",
+    .fun = prim_parseDrvName,
+});
 
 static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2212,6 +3230,18 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a
     mkInt(v, compareVersions(version1, version2));
 }
 
+static RegisterPrimOp primop_compareVersions({
+    .name = "__compareVersions",
+    .args = {"s1", "s2"},
+    .doc = R"(
+      Compare two strings representing versions and return `-1` if
+      version *s1* is older than version *s2*, `0` if they are the same,
+      and `1` if *s1* is newer than *s2*. The version comparison
+      algorithm is the same as the one used by [`nix-env
+      -u`](../command-ref/nix-env.md#operation---upgrade).
+    )",
+    .fun = prim_compareVersions,
+});
 
 static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2232,6 +3262,17 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
     }
 }
 
+static RegisterPrimOp primop_splitVersion({
+    .name = "__splitVersion",
+    .args = {"s"},
+    .doc = R"(
+      Split a string representing a version into its components, by the
+      same version splitting logic underlying the version comparison in
+      [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade).
+    )",
+    .fun = prim_splitVersion,
+});
+
 
 /*************************************************************
  * Primop registration
@@ -2282,15 +3323,6 @@ void EvalState::createBaseEnv()
     mkNull(v);
     addConstant("null", v);
 
-    auto vThrow = addPrimOp("throw", 1, prim_throw);
-
-    auto addPurityError = [&](const std::string & name) {
-        Value * v2 = allocValue();
-        mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name));
-        mkApp(v, *vThrow, *v2);
-        addConstant(name, v);
-    };
-
     if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
@@ -2325,115 +3357,24 @@ void EvalState::createBaseEnv()
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
     }
-    addPrimOp("__typeOf", 1, prim_typeOf);
-    addPrimOp("isNull", 1, prim_isNull);
-    addPrimOp("__isFunction", 1, prim_isFunction);
-    addPrimOp("__isString", 1, prim_isString);
-    addPrimOp("__isInt", 1, prim_isInt);
-    addPrimOp("__isFloat", 1, prim_isFloat);
-    addPrimOp("__isBool", 1, prim_isBool);
-    addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
-    addPrimOp("__tryEval", 1, prim_tryEval);
-    addPrimOp("__getEnv", 1, prim_getEnv);
-
-    // Strictness
-    addPrimOp("__seq", 2, prim_seq);
-    addPrimOp("__deepSeq", 2, prim_deepSeq);
-
-    // Debugging
-    addPrimOp("__trace", 2, prim_trace);
 
     // Paths
-    addPrimOp("__toPath", 1, prim_toPath);
-    if (evalSettings.pureEval)
-        addPurityError("__storePath");
-    else
-        addPrimOp("__storePath", 1, prim_storePath);
-    addPrimOp("__pathExists", 1, prim_pathExists);
-    addPrimOp("baseNameOf", 1, prim_baseNameOf);
-    addPrimOp("dirOf", 1, prim_dirOf);
-    addPrimOp("__readFile", 1, prim_readFile);
-    addPrimOp("__readDir", 1, prim_readDir);
+    addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__findFile", 2, prim_findFile);
-    addPrimOp("__hashFile", 2, prim_hashFile);
-
-    // Creating files
-    addPrimOp("__toXML", 1, prim_toXML);
-    addPrimOp("__toJSON", 1, prim_toJSON);
-    addPrimOp("__fromJSON", 1, prim_fromJSON);
-    addPrimOp("__toFile", 2, prim_toFile);
-    addPrimOp("__filterSource", 2, prim_filterSource);
-    addPrimOp("__path", 1, prim_path);
 
     // Sets
-    addPrimOp("__attrNames", 1, prim_attrNames);
-    addPrimOp("__attrValues", 1, prim_attrValues);
-    addPrimOp("__getAttr", 2, prim_getAttr);
     addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos);
-    addPrimOp("__hasAttr", 2, prim_hasAttr);
-    addPrimOp("__isAttrs", 1, prim_isAttrs);
-    addPrimOp("removeAttrs", 2, prim_removeAttrs);
-    addPrimOp("__listToAttrs", 1, prim_listToAttrs);
-    addPrimOp("__intersectAttrs", 2, prim_intersectAttrs);
     addPrimOp("__catAttrs", 2, prim_catAttrs);
-    addPrimOp("__functionArgs", 1, prim_functionArgs);
     addPrimOp("__mapAttrs", 2, prim_mapAttrs);
 
     // Lists
-    addPrimOp("__isList", 1, prim_isList);
-    addPrimOp("__elemAt", 2, prim_elemAt);
-    addPrimOp("__head", 1, prim_head);
-    addPrimOp("__tail", 1, prim_tail);
-    addPrimOp("map", 2, prim_map);
-    addPrimOp("__filter", 2, prim_filter);
-    addPrimOp("__elem", 2, prim_elem);
-    addPrimOp("__concatLists", 1, prim_concatLists);
-    addPrimOp("__length", 1, prim_length);
-    addPrimOp("__foldl'", 3, prim_foldlStrict);
-    addPrimOp("__any", 2, prim_any);
-    addPrimOp("__all", 2, prim_all);
-    addPrimOp("__genList", 2, prim_genList);
-    addPrimOp("__sort", 2, prim_sort);
     addPrimOp("__partition", 2, prim_partition);
     addPrimOp("__concatMap", 2, prim_concatMap);
 
-    // Integer arithmetic
-    addPrimOp("__add", 2, prim_add);
-    addPrimOp("__sub", 2, prim_sub);
-    addPrimOp("__mul", 2, prim_mul);
-    addPrimOp("__div", 2, prim_div);
-    addPrimOp("__bitAnd", 2, prim_bitAnd);
-    addPrimOp("__bitOr", 2, prim_bitOr);
-    addPrimOp("__bitXor", 2, prim_bitXor);
-    addPrimOp("__lessThan", 2, prim_lessThan);
-
-    // String manipulation
-    addPrimOp("toString", 1, prim_toString);
-    addPrimOp("__substring", 3, prim_substring);
-    addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hashString", 2, prim_hashString);
-    addPrimOp("__match", 2, prim_match);
-    addPrimOp("__split", 2, prim_split);
-    addPrimOp("__concatStringsSep", 2, prim_concatStringSep);
-    addPrimOp("__replaceStrings", 3, prim_replaceStrings);
-
-    // Versions
-    addPrimOp("__parseDrvName", 1, prim_parseDrvName);
-    addPrimOp("__compareVersions", 2, prim_compareVersions);
-    addPrimOp("__splitVersion", 1, prim_splitVersion);
-
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
-    addPrimOp("placeholder", 1, prim_placeholder);
-
-    /* Add a wrapper around the derivation primop that computes the
-       `drvPath' and `outPath' attributes lazily. */
-    string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
-    sDerivationNix = symbols.create(path);
-    evalFile(path, v);
-    addConstant("derivation", v);
 
     /* Add a value containing the current Nix expression search path. */
     mkList(v, searchPath.size());
@@ -2458,6 +3399,13 @@ void EvalState::createBaseEnv()
                     .doc = primOp.doc,
                 });
 
+    /* Add a wrapper around the derivation primop that computes the
+       `drvPath' and `outPath' attributes lazily. */
+    string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
+    sDerivationNix = symbols.create(path);
+    evalFile(path, v);
+    addConstant("derivation", v);
+
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
     baseEnv.values[0]->attrs->sort();
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 0dbf4ae1d..06e8304b8 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -226,18 +226,187 @@ static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Va
     fetch(state, pos, args, v, "fetchurl", false, "");
 }
 
+static RegisterPrimOp primop_fetchurl({
+    .name = "__fetchurl",
+    .args = {"url"},
+    .doc = R"(
+      Download the specified URL and return the path of the downloaded
+      file. This function is not available if [restricted evaluation
+      mode](../command-ref/conf-file.md) is enabled.
+    )",
+    .fun = prim_fetchurl,
+});
+
 static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     fetch(state, pos, args, v, "fetchTarball", true, "source");
 }
 
+static RegisterPrimOp primop_fetchTarball({
+    .name = "fetchTarball",
+    .args = {"args"},
+    .doc = R"(
+      Download the specified URL, unpack it and return the path of the
+      unpacked tree. The file must be a tape archive (`.tar`) compressed
+      with `gzip`, `bzip2` or `xz`. The top-level path component of the
+      files in the tarball is removed, so it is best if the tarball
+      contains a single directory at top level. The typical use of the
+      function is to obtain external Nix expression dependencies, such as
+      a particular version of Nixpkgs, e.g.
+
+      ```nix
+      with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
+
+      stdenv.mkDerivation { … }
+      ```
+
+      The fetched tarball is cached for a certain amount of time (1 hour
+      by default) in `~/.cache/nix/tarballs/`. You can change the cache
+      timeout either on the command line with `--option tarball-ttl number
+      of seconds` or in the Nix configuration file with this option: ` 
+      number of seconds to cache `.
+
+      Note that when obtaining the hash with ` nix-prefetch-url ` the
+      option `--unpack` is required.
+
+      This function can also verify the contents against a hash. In that
+      case, the function takes a set instead of a URL. The set requires
+      the attribute `url` and the attribute `sha256`, e.g.
+
+      ```nix
+      with import (fetchTarball {
+        url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
+        sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
+      }) {};
+
+      stdenv.mkDerivation { … }
+      ```
+
+      This function is not available if [restricted evaluation
+      mode](../command-ref/conf-file.md) is enabled.
+    )",
+    .fun = prim_fetchTarball,
+});
+
 static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
 {
     fetchTree(state, pos, args, v, "git", true);
 }
 
-static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
-static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
-static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
+static RegisterPrimOp primop_fetchGit({
+    .name = "fetchGit",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a path from git. *args* can be a URL, in which case the HEAD
+      of the repo at that URL is fetched. Otherwise, it can be an
+      attribute with the following attributes (all except `url` optional):
+
+        - url  
+          The URL of the repo.
+
+        - name  
+          The name of the directory the repo should be exported to in the
+          store. Defaults to the basename of the URL.
+
+        - rev  
+          The git revision to fetch. Defaults to the tip of `ref`.
+
+        - ref  
+          The git ref to look for the requested revision under. This is
+          often a branch or tag name. Defaults to `HEAD`.
+
+          By default, the `ref` value is prefixed with `refs/heads/`. As
+          of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
+          with `refs/`.
+
+        - submodules  
+          A Boolean parameter that specifies whether submodules should be
+          checked out. Defaults to `false`.
+
+      Here are some examples of how to use `fetchGit`.
+
+        - To fetch a private repository over SSH:
+
+          ```nix
+          builtins.fetchGit {
+            url = "git@github.com:my-secret/repository.git";
+            ref = "master";
+            rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+          }
+          ```
+
+        - To fetch an arbitrary reference:
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/NixOS/nix.git";
+            ref = "refs/heads/0.5-release";
+          }
+          ```
+
+        - If the revision you're looking for is in the default branch of
+          the git repository you don't strictly need to specify the branch
+          name in the `ref` attribute.
+
+          However, if the revision you're looking for is in a future
+          branch for the non-default branch you will need to specify the
+          the `ref` attribute as well.
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+            ref = "1.11-maintenance";
+          }
+          ```
+
+          > **Note**
+          > 
+          > It is nice to always specify the branch which a revision
+          > belongs to. Without the branch being specified, the fetcher
+          > might fail if the default branch changes. Additionally, it can
+          > be confusing to try a commit from a non-default branch and see
+          > the fetch fail. If the branch is specified the fault is much
+          > more obvious.
+
+        - If the revision you're looking for is in the default branch of
+          the git repository you may omit the `ref` attribute.
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+          }
+          ```
+
+        - To fetch a specific tag:
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            ref = "refs/tags/1.9";
+          }
+          ```
+
+        - To fetch the latest version of a remote branch:
+
+          ```nix
+          builtins.fetchGit {
+            url = "ssh://git@github.com/nixos/nix.git";
+            ref = "master";
+          }
+          ```
+
+          > **Note**
+          > 
+          > Nix will refetch the branch in accordance with
+          > the option `tarball-ttl`.
+
+          > **Note**
+          > 
+          > This behavior is disabled in *Pure evaluation mode*.
+    )",
+    .fun = prim_fetchGit,
+});
 
 }

From 0f314f3c2594e80322c675b70a61dcfda11bf423 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 14:49:30 +0200
Subject: [PATCH 134/882] Generate builtins section of the manual

---
 .gitignore                                    |  2 ++
 doc/manual/generate-builtins.jq               |  6 ++++++
 doc/manual/local.mk                           | 11 ++++++++--
 .../{builtins.md => builtins-prefix.md}       |  0
 src/nix/main.cc                               | 20 ++++++++++++++++++-
 5 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 doc/manual/generate-builtins.jq
 rename doc/manual/src/expressions/{builtins.md => builtins-prefix.md} (100%)

diff --git a/.gitignore b/.gitignore
index 44dbaa5d7..0ea27c8c8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,8 +27,10 @@ perl/Makefile.config
 /doc/manual/*.8
 /doc/manual/nix.json
 /doc/manual/conf-file.json
+/doc/manual/builtins.json
 /doc/manual/src/command-ref/nix.md
 /doc/manual/src/command-ref/conf-file.md
+/doc/manual/src/expressions/builtins.md
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/generate-builtins.jq b/doc/manual/generate-builtins.jq
new file mode 100644
index 000000000..c38799479
--- /dev/null
+++ b/doc/manual/generate-builtins.jq
@@ -0,0 +1,6 @@
+. | to_entries | sort_by(.key) | map(
+  "  - `builtins." + .key + "` "
+  + (.value.args | map("*" + . + "*") | join(" "))
+  + "  \n\n"
+  + (.value.doc | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
+) | join("")
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index dcc02d538..297a73414 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -32,15 +32,22 @@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.jq
 	jq -r -f doc/manual/generate-options.jq $< >> $@
 
 $(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(bindir)/nix dump-args > $@
+	$(trace-gen) $(bindir)/nix __dump-args > $@
 
 $(d)/conf-file.json: $(bindir)/nix
 	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@
 
+$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.jq $(d)/src/expressions/builtins-prefix.md
+	cat doc/manual/src/expressions/builtins-prefix.md > $@
+	jq -r -f doc/manual/generate-builtins.jq $< >> $@
+
+$(d)/builtins.json: $(bindir)/nix
+	$(trace-gen) $(bindir)/nix __dump-builtins > $@
+
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/command-ref/nix.md $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
 	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
 	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
 
diff --git a/doc/manual/src/expressions/builtins.md b/doc/manual/src/expressions/builtins-prefix.md
similarity index 100%
rename from doc/manual/src/expressions/builtins.md
rename to doc/manual/src/expressions/builtins-prefix.md
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 0c4fd46fd..210327927 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -179,11 +179,29 @@ void mainWrapped(int argc, char * * argv)
 
     NixArgs args;
 
-    if (argc == 2 && std::string(argv[1]) == "dump-args") {
+    if (argc == 2 && std::string(argv[1]) == "__dump-args") {
         std::cout << args.toJSON().dump() << "\n";
         return;
     }
 
+    if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+        EvalState state({}, openStore("ssh://foo.invalid/"));
+        auto res = nlohmann::json::object();
+        auto builtins = state.baseEnv.values[0]->attrs;
+        for (auto & builtin : *builtins) {
+            auto b = nlohmann::json::object();
+            if (builtin.value->type != tPrimOp) continue;
+            auto primOp = builtin.value->primOp;
+            if (!primOp->doc) continue;
+            b["arity"] = primOp->arity;
+            b["args"] = primOp->args;
+            b["doc"] = trim(stripIndentation(primOp->doc));
+            res[(std::string) builtin.name] = std::move(b);
+        }
+        std::cout << res.dump() << "\n";
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {

From d0690bc311ee4969fc35604fed0fe819d4938704 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 18:10:33 +0200
Subject: [PATCH 135/882] nix repl ':doc': Render using lowdown

---
 src/nix/local.mk    |  5 +++--
 src/nix/markdown.cc | 50 +++++++++++++++++++++++++++++++++++++++++++++
 src/nix/markdown.hh |  7 +++++++
 src/nix/repl.cc     | 15 ++++++++++----
 4 files changed, 71 insertions(+), 6 deletions(-)
 create mode 100644 src/nix/markdown.cc
 create mode 100644 src/nix/markdown.hh

diff --git a/src/nix/local.mk b/src/nix/local.mk
index b057b7cc6..8ed8eaef7 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,11 +15,12 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
+# -fpermissive is needed by lowdown.
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -fpermissive
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -llowdown
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
diff --git a/src/nix/markdown.cc b/src/nix/markdown.cc
new file mode 100644
index 000000000..40788a42f
--- /dev/null
+++ b/src/nix/markdown.cc
@@ -0,0 +1,50 @@
+#include "markdown.hh"
+#include "util.hh"
+#include "finally.hh"
+
+#include <sys/queue.h>
+extern "C" {
+#include <lowdown.h>
+}
+
+namespace nix {
+
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    struct lowdown_opts opts {
+        .type = LOWDOWN_TERM,
+        .maxdepth = 20,
+        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .hmargin = 0,
+        .vmargin = 0,
+        .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
+        .oflags = 0,
+    };
+
+    auto doc = lowdown_doc_new(&opts);
+    if (!doc)
+        throw Error("cannot allocate Markdown document");
+    Finally freeDoc([&]() { lowdown_doc_free(doc); });
+
+    size_t maxn = 0;
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    if (!node)
+        throw Error("cannot parse Markdown document");
+    Finally freeNode([&]() { lowdown_node_free(node); });
+
+    auto renderer = lowdown_term_new(&opts);
+    if (!renderer)
+        throw Error("cannot allocate Markdown renderer");
+    Finally freeRenderer([&]() { lowdown_term_free(renderer); });
+
+    auto buf = lowdown_buf_new(16384);
+    if (!buf)
+        throw Error("cannot allocate Markdown output buffer");
+    Finally freeBuffer([&]() { lowdown_buf_free(buf); });
+
+    lowdown_term_rndr(buf, nullptr, renderer, node);
+
+    return std::string(buf->data, buf->size);
+}
+
+}
diff --git a/src/nix/markdown.hh b/src/nix/markdown.hh
new file mode 100644
index 000000000..78320fcf5
--- /dev/null
+++ b/src/nix/markdown.hh
@@ -0,0 +1,7 @@
+#include "types.hh"
+
+namespace nix {
+
+std::string renderMarkdownToTerminal(std::string_view markdown);
+
+}
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index d370ca767..c7f8af742 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -32,6 +32,7 @@ extern "C" {
 #include "globals.hh"
 #include "command.hh"
 #include "finally.hh"
+#include "markdown.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -518,10 +519,16 @@ bool NixRepl::processLine(string line)
             while (v2->type == tPrimOpApp)
                 v2 = v2->primOpApp.left;
             if (v2->primOp->doc) {
-                // FIXME: format markdown.
-                if (!v2->primOp->args.empty())
-                    std::cout << fmt("Arguments: %s\n\n", concatStringsSep(" ", v2->primOp->args));
-                std::cout << trim(stripIndentation(v2->primOp->doc)) << "\n";
+                auto args = v2->primOp->args;
+                for (auto & arg : args)
+                    arg = "*" + arg + "*";
+
+                auto markdown =
+                    "**Synopsis:** `builtins." + (std::string) v2->primOp->name + "` "
+                    + concatStringsSep(" ", args) + "\n\n"
+                    + trim(stripIndentation(v2->primOp->doc));
+
+                std::cout << renderMarkdownToTerminal(markdown);
             } else
                 throw Error("builtin function '%s' does not have documentation", v2->primOp->name);
         } else

From 6a67e57019f39e43866866661df17394c168d29b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 18:54:16 +0200
Subject: [PATCH 136/882] Add DummyStore (dummy://)

DummyStore does not allow building or adding paths. This is useful for
evaluation tests when you don't want to initialize a "proper" store.
---
 src/libstore/dummy-store.cc | 59 +++++++++++++++++++++++++++++++++++++
 src/nix/main.cc             |  2 +-
 tests/lang.sh               |  1 +
 3 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 src/libstore/dummy-store.cc

diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
new file mode 100644
index 000000000..7a5744bc1
--- /dev/null
+++ b/src/libstore/dummy-store.cc
@@ -0,0 +1,59 @@
+#include "store-api.hh"
+
+namespace nix {
+
+static std::string uriScheme = "dummy://";
+
+struct DummyStore : public Store
+{
+    DummyStore(const Params & params)
+        : Store(params)
+    { }
+
+    string getUri() override
+    {
+        return uriScheme;
+    }
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
+    {
+        callback(nullptr);
+    }
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+    { unsupported("queryPathFromHashPart"); }
+
+    void addToStore(const ValidPathInfo & info, Source & source,
+        RepairFlag repair, CheckSigsFlag checkSigs) override
+    { unsupported("addToStore"); }
+
+    StorePath addToStore(const string & name, const Path & srcPath,
+        FileIngestionMethod method, HashType hashAlgo,
+        PathFilter & filter, RepairFlag repair) override
+    { unsupported("addToStore"); }
+
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override
+    { unsupported("addTextToStore"); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { unsupported("narFromPath"); }
+
+    void ensurePath(const StorePath & path) override
+    { unsupported("ensurePath"); }
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override
+    { unsupported("buildDerivation"); }
+};
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (uri != uriScheme) return nullptr;
+    return std::make_shared<DummyStore>(params);
+});
+
+}
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 210327927..e9479f564 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -185,7 +185,7 @@ void mainWrapped(int argc, char * * argv)
     }
 
     if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
-        EvalState state({}, openStore("ssh://foo.invalid/"));
+        EvalState state({}, openStore("dummy://"));
         auto res = nlohmann::json::object();
         auto builtins = state.baseEnv.values[0]->attrs;
         for (auto & builtin : *builtins) {
diff --git a/tests/lang.sh b/tests/lang.sh
index c797a2a74..61bb444ba 100644
--- a/tests/lang.sh
+++ b/tests/lang.sh
@@ -1,6 +1,7 @@
 source common.sh
 
 export TEST_VAR=foo # for eval-okay-getenv.nix
+export NIX_REMOTE=dummy://
 
 nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
 (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)

From d9a8619762fa56bca8ddec5d9bcd5c411adb5b95 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 19:15:17 +0200
Subject: [PATCH 137/882] Don't barf if corepkgs is in the store but not a
 valid path

This can happen when using a dummy store (or indeed any non-local store).
---
 src/libexpr/eval.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 6ac8bbc9f..a5ebd1bf0 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -381,10 +381,14 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
             auto path = r.second;
 
             if (store->isInStore(r.second)) {
-                StorePathSet closure;
-                store->computeFSClosure(store->toStorePath(r.second).first, closure);
-                for (auto & path : closure)
-                    allowedPaths->insert(store->printStorePath(path));
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(r.second).first, closure);
+                    for (auto & path : closure)
+                        allowedPaths->insert(store->printStorePath(path));
+                } catch (InvalidPath &) {
+                    allowedPaths->insert(r.second);
+                }
             } else
                 allowedPaths->insert(r.second);
         }

From 59979e705352abb1624d3427c2c7145ed43b1b84 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 24 Aug 2020 18:10:58 +0000
Subject: [PATCH 138/882] Fix bad debug format string

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ba28e78c8..6baaa31d9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3850,7 +3850,7 @@ void DerivationGoal::registerOutputs()
            something like that. */
         canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
 
-        debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath);
+        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
 
         /* Pass blank Sink as we are not ready to hash data at this stage. */
         NullSink blank;

From e0b0e18905835fdc8ccbbf1c0f5d016d9f466187 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 23 Aug 2020 15:27:30 +0000
Subject: [PATCH 139/882] Add constructor for BasicDerivation -> Derivation

---
 src/libstore/derivations.hh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index b09480e1e..2ea4178c0 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -100,7 +100,7 @@ struct BasicDerivation
     StringPairs env;
     std::string name;
 
-    BasicDerivation() { }
+    BasicDerivation() = default;
     virtual ~BasicDerivation() { };
 
     bool isBuiltin() const;
@@ -127,7 +127,8 @@ struct Derivation : BasicDerivation
     std::string unparse(const Store & store, bool maskOutputs,
         std::map<std::string, StringSet> * actualInputs = nullptr) const;
 
-    Derivation() { }
+    Derivation() = default;
+    Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 };
 
 

From 8eb73a87245acf9d93dc401831b629981864fa58 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 22 Aug 2020 20:44:47 +0000
Subject: [PATCH 140/882] CA derivations that depend on other CA derivations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
---
 src/libstore/build.cc       | 51 +++++++++++++++++++++++++++++++----
 src/libstore/derivations.cc | 53 +++++++++++++++++++++++++++++++++++++
 src/libstore/derivations.hh | 18 +++++++++++++
 src/libstore/local-store.cc | 27 +++++++++++++++++--
 tests/content-addressed.nix | 48 ++++++++++++++++++++++++---------
 tests/content-addressed.sh  | 20 +++++++-------
 6 files changed, 189 insertions(+), 28 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6baaa31d9..f5256bf87 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -984,6 +984,8 @@ private:
     void tryLocalBuild();
     void buildDone();
 
+    void resolvedFinished();
+
     /* Is the build hook willing to perform the build? */
     HookReply tryBuildHook();
 
@@ -1451,8 +1453,39 @@ void DerivationGoal::inputsRealised()
     /* Determine the full set of input paths. */
 
     /* First, the input derivations. */
-    if (useDerivation)
-        for (auto & [depDrvPath, wantedDepOutputs] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+    if (useDerivation) {
+        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
+
+        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
+            /* We are be able to resolve this derivation based on the
+               now-known results of dependencies. If so, we become a stub goal
+               aliasing that resolved derivation goal */
+            Derivation drvResolved { fullDrv.resolve(worker.store) };
+
+            auto pathResolved = writeDerivation(worker.store, drvResolved);
+            /* Add to memotable to speed up downstream goal's queries with the
+               original derivation. */
+            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
+
+            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
+                worker.store.printStorePath(drvPath),
+                worker.store.printStorePath(pathResolved));
+            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
+                Logger::Fields {
+                       worker.store.printStorePath(drvPath),
+                       worker.store.printStorePath(pathResolved),
+                   });
+
+            auto resolvedGoal = worker.makeDerivationGoal(
+                pathResolved, wantedOutputs,
+                buildMode == bmRepair ? bmRepair : bmNormal);
+            addWaitee(resolvedGoal);
+
+            state = &DerivationGoal::resolvedFinished;
+            return;
+        }
+
+        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
@@ -1472,6 +1505,7 @@ void DerivationGoal::inputsRealised()
                         worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
             }
         }
+    }
 
     /* Second, the input sources. */
     worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@@ -1893,6 +1927,9 @@ void DerivationGoal::buildDone()
     done(BuildResult::Built);
 }
 
+void DerivationGoal::resolvedFinished() {
+    done(BuildResult::Built);
+}
 
 HookReply DerivationGoal::tryBuildHook()
 {
@@ -2065,7 +2102,7 @@ void linkOrCopy(const Path & from, const Path & to)
            file (e.g. 32000 of ext3), which is quite possible after a
            'nix-store --optimise'. FIXME: actually, why don't we just
            bind-mount in this case?
-           
+
            It can also fail with EPERM in BeegFS v7 and earlier versions
            which don't allow hard-links to other directories */
         if (errno != EMLINK && errno != EPERM)
@@ -4248,10 +4285,14 @@ void DerivationGoal::registerOutputs()
     {
         ValidPathInfos infos2;
         for (auto & [outputName, newInfo] : infos) {
-            /* FIXME: we will want to track this mapping in the DB whether or
-               not we have a drv file. */
             if (useDerivation)
                 worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
+            else {
+                /* Once a floating CA derivations reaches this point, it must
+                   already be resolved, drvPath the basic derivation path, and
+                   a file existsing at that path for sake of the DB's foreign key. */
+                assert(drv->type() != DerivationType::CAFloating);
+            }
             infos2.push_back(newInfo);
         }
         worker.store.registerValidPaths(infos2);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 34541227b..d96d4083d 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -672,4 +672,57 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
     return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
 }
 
+
+// N.B. Outputs are left unchanged
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
+
+    debug("Rewriting the derivation");
+
+    for (auto &rewrite: rewrites) {
+        debug("rewriting %s as %s", rewrite.first, rewrite.second);
+    }
+
+    drv.builder = rewriteStrings(drv.builder, rewrites);
+    for (auto & arg: drv.args) {
+        arg = rewriteStrings(arg, rewrites);
+    }
+
+    StringPairs newEnv;
+    for (auto & envVar: drv.env) {
+        auto envName = rewriteStrings(envVar.first, rewrites);
+        auto envValue = rewriteStrings(envVar.second, rewrites);
+        newEnv.emplace(envName, envValue);
+    }
+    drv.env = newEnv;
+}
+
+
+Sync<DrvPathResolutions> drvPathResolutions;
+
+BasicDerivation Derivation::resolve(Store & store) {
+    BasicDerivation resolved { *this };
+
+    // Input paths that we'll want to rewrite in the derivation
+    StringMap inputRewrites;
+
+    for (auto & input : inputDrvs) {
+        auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
+        StringSet newOutputNames;
+        for (auto & outputName : input.second) {
+            auto actualPathOpt = inputDrvOutputs.at(outputName);
+            if (!actualPathOpt)
+                throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first));
+            auto actualPath = *actualPathOpt;
+            inputRewrites.emplace(
+                downstreamPlaceholder(store, input.first, outputName),
+                store.printStorePath(actualPath));
+            resolved.inputSrcs.insert(std::move(actualPath));
+        }
+    }
+
+    rewriteDerivation(store, resolved, inputRewrites);
+
+    return resolved;
+}
+
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 2ea4178c0..0bb565e8a 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -4,6 +4,7 @@
 #include "types.hh"
 #include "hash.hh"
 #include "content-address.hh"
+#include "sync.hh"
 
 #include <map>
 #include <variant>
@@ -127,6 +128,13 @@ struct Derivation : BasicDerivation
     std::string unparse(const Store & store, bool maskOutputs,
         std::map<std::string, StringSet> * actualInputs = nullptr) const;
 
+    /* Return the underlying basic derivation but with
+
+       1. input drv outputs moved to input sources.
+
+       2. placeholders replaced with realized input store paths. */
+    BasicDerivation resolve(Store & store);
+
     Derivation() = default;
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 };
@@ -187,6 +195,16 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes;
 
 extern DrvHashes drvHashes; // FIXME: global, not thread-safe
 
+/* Memoisation of `readDerivation(..).resove()`. */
+typedef std::map<
+    StorePath,
+    std::optional<StorePath>
+> DrvPathResolutions;
+
+// FIXME: global, though at least thread-safe.
+// FIXME: arguably overlaps with hashDerivationModulo memo table.
+extern Sync<DrvPathResolutions> drvPathResolutions;
+
 bool wantOutput(const string & output, const std::set<string> & wanted);
 
 struct Source;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0086bb13e..e51d127b3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -803,13 +803,36 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
 {
+    auto path = path_;
     std::map<std::string, std::optional<StorePath>> outputs;
-    BasicDerivation drv = readDerivation(path);
+    Derivation drv = readDerivation(path);
     for (auto & [outName, _] : drv.outputs) {
         outputs.insert_or_assign(outName, std::nullopt);
     }
+    bool haveCached = false;
+    {
+        auto resolutions = drvPathResolutions.lock();
+        auto resolvedPathOptIter = resolutions->find(path);
+        if (resolvedPathOptIter != resolutions->end()) {
+            auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
+            if (resolvedPathOpt)
+                path = *resolvedPathOpt;
+            haveCached = true;
+        }
+    }
+    /* can't just use else-if instead of `!haveCached` because we need to unlock
+       `drvPathResolutions` before it is locked in `Derivation::resolve`. */
+    if (!haveCached && drv.type() == DerivationType::CAFloating) {
+        /* Resolve drv and use that path instead. */
+        auto pathResolved = writeDerivation(*this, drv.resolve(*this));
+        /* Store in memo table. */
+        /* FIXME: memo logic should not be local-store specific, should have
+           wrapper-method instead. */
+        drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
+        path = std::move(pathResolved);
+    }
     return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
         auto state(_state.lock());
 
diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 586e4cba6..a46c21164 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -4,16 +4,40 @@ with import ./config.nix;
 # A simple content-addressed derivation.
 # The derivation can be arbitrarily modified by passing a different `seed`,
 # but the output will always be the same
-mkDerivation {
-  name = "simple-content-addressed";
-  buildCommand = ''
-    set -x
-    echo "Building a CA derivation"
-    echo "The seed is ${toString seed}"
-    mkdir -p $out
-    echo "Hello World" > $out/hello
-  '';
-  __contentAddressed = true;
-  outputHashMode = "recursive";
-  outputHashAlgo = "sha256";
+rec {
+  root = mkDerivation {
+    name = "simple-content-addressed";
+    buildCommand = ''
+      set -x
+      echo "Building a CA derivation"
+      echo "The seed is ${toString seed}"
+      mkdir -p $out
+      echo "Hello World" > $out/hello
+    '';
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  };
+  dependent = mkDerivation {
+    name = "dependent";
+    buildCommand = ''
+      echo "building a dependent derivation"
+      mkdir -p $out
+      echo ${root}/hello > $out/dep
+    '';
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  };
+  transitivelyDependent = mkDerivation {
+    name = "transitively-dependent";
+    buildCommand = ''
+      echo "building transitively-dependent"
+      cat ${dependent}/dep
+      echo ${dependent} > $out
+    '';
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  };
 }
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 2968f3a8c..522310585 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -2,15 +2,17 @@
 
 source common.sh
 
-clearStore
-clearCache
-
-export REMOTE_STORE=file://$cacheDir
-
-drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1)
+drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A root --arg seed 1)
 nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
 
-out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link)
-out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link)
+testDerivation () {
+    local derivationPath=$1
+    local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
+    local out1=$(nix-build "${commonArgs[@]}" --arg seed 1)
+    local out2=$(nix-build "${commonArgs[@]}" --arg seed 2)
+    test $out1 == $out2
+}
 
-test $out1 == $out2
+testDerivation root
+testDerivation dependent
+testDerivation transitivelyDependent

From b42789f01382c49256b2b5073098f5fb01841950 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Aug 2020 21:13:39 +0200
Subject: [PATCH 141/882] Fix clang build

---
 flake.lock       | 20 +++++++++++++++++++-
 flake.nix        |  7 ++++++-
 src/nix/local.mk |  2 +-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 74326d294..f4368b170 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,22 @@
 {
   "nodes": {
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1598296217,
+        "narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=",
+        "owner": "edolstra",
+        "repo": "lowdown",
+        "rev": "c7a4e715af1e233080842db82d15b261cb74cb28",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "no-structs-in-anonymous-unions",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1591633336,
@@ -17,10 +34,11 @@
     },
     "root": {
       "inputs": {
+        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs"
       }
     }
   },
   "root": "root",
-  "version": 6
+  "version": 7
 }
diff --git a/flake.nix b/flake.nix
index f97197cce..ea0ad7b8a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,8 +2,9 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
+  inputs.lowdown-src = { url = "github:edolstra/lowdown/no-structs-in-anonymous-unions"; flake = false; };
 
-  outputs = { self, nixpkgs }:
+  outputs = { self, nixpkgs, lowdown-src }:
 
     let
 
@@ -188,10 +189,14 @@
         lowdown = with final; stdenv.mkDerivation {
           name = "lowdown-0.7.1";
 
+          /*
           src = fetchurl {
             url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.1.tar.gz;
             hash = "sha512-1daoAQfYD0LdhK6aFhrSQvadjc5GsSPBZw0fJDb+BEHYMBLjqiUl2A7H8N+l0W4YfGKqbsPYSrCy4vct+7U6FQ==";
           };
+          */
+
+          src = lowdown-src;
 
           outputs = [ "out" "dev" ];
 
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 8ed8eaef7..cfe97b0f0 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -16,7 +16,7 @@ nix_SOURCES := \
   $(wildcard src/nix-store/*.cc) \
 
 # -fpermissive is needed by lowdown.
-nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -fpermissive
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil
 

From f5219f8d84af57c68f6eb129260bf9f0805b9b47 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Aug 2020 10:33:41 +0200
Subject: [PATCH 142/882] Fix perlBindings job

---
 flake.nix | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index ea0ad7b8a..3f67bfd40 100644
--- a/flake.nix
+++ b/flake.nix
@@ -73,9 +73,6 @@
             openssl pkgconfig sqlite
             libarchive
             boost
-            (if lib.versionAtLeast lib.version "20.03pre"
-             then nlohmann_json
-             else nlohmann_json.override { multipleHeaders = true; })
             nlohmann_json
 
             # Tests
@@ -171,6 +168,7 @@
                 pkgconfig
                 pkgs.perl
                 boost
+                nlohmann_json
               ]
               ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 

From 24b1c2c66b2f7ee8bbb9f62f4d914eb87a998999 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Aug 2020 10:51:14 +0200
Subject: [PATCH 143/882] Fix tests

---
 src/libutil/tests/config.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index 9465fba3f..c5abefe11 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -1,9 +1,9 @@
-#include "json.hh"
 #include "config.hh"
 #include "args.hh"
 
 #include <sstream>
 #include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 

From b8416779e3c17be819a27f1527a53d83c7eccbcf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Aug 2020 11:16:45 +0200
Subject: [PATCH 144/882] Document some primops

---
 src/libexpr/primops.cc | 111 +++++++++++++++++++++++++++++------------
 1 file changed, 80 insertions(+), 31 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 78892053a..cdc526de3 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1053,7 +1053,7 @@ static RegisterPrimOp primop_toPath({
     .name = "__toPath",
     .args = {"s"},
     .doc = R"(
-      DEPRECATED. Use `/. + "/path"` to convert a string into an absolute
+      **DEPRECATED.** Use `/. + "/path"` to convert a string into an absolute
       path. For relative paths, use `./. + "/path"`.
     )",
     .fun = prim_toPath,
@@ -1090,6 +1090,23 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
     mkString(v, path, context);
 }
 
+static RegisterPrimOp primop_storePath({
+    .name = "__storePath",
+    .args = {"path"},
+    .doc = R"(
+      This function allows you to define a dependency on an already
+      existing store path. For example, the derivation attribute `src
+      = builtins.storePath /nix/store/f1d18v1y…-source` causes the
+      derivation to depend on the specified path, which must exist or
+      be substitutable. Note that this differs from a plain path
+      (e.g. `src = /nix/store/f1d18v1y…-source`) in that the latter
+      causes the path to be *copied* again to the Nix store, resulting
+      in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
+
+      This function is not available in pure evaluation mode.
+    )",
+    .fun = prim_storePath,
+});
 
 static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2022,9 +2039,6 @@ static RegisterPrimOp primop_listToAttrs({
     .fun = prim_listToAttrs,
 });
 
-/* Return the right-biased intersection of two sets as1 and as2,
-   i.e. a set that contains every attribute from as2 that is also a
-   member of as1. */
 static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], pos);
@@ -2049,13 +2063,6 @@ static RegisterPrimOp primop_intersectAttrs({
     .fun = prim_intersectAttrs,
 });
 
-/* Collect each attribute named `attr' from a list of attribute sets.
-   Sets that don't contain the named attribute are ignored.
-
-   Example:
-     catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}]
-     => [1 2]
-*/
 static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos));
@@ -2077,19 +2084,23 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
         v.listElems()[n] = res[n];
 }
 
-/* Return a set containing the names of the formal arguments expected
-   by the function `f'.  The value of each attribute is a Boolean
-   denoting whether the corresponding argument has a default value.  For instance,
+static RegisterPrimOp primop_catAttrs({
+    .name = "__catAttrs",
+    .args = {"attr", "list"},
+    .doc = R"(
+      Collect each attribute named *attr* from a list of attribute
+      sets.  Attrsets that don't contain the named attribute are
+      ignored. For example,
 
-      functionArgs ({ x, y ? 123}: ...)
-   => { x = false; y = true; }
+      ```nix
+      builtins.catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}]
+      ```
 
-   "Formal argument" here refers to the attributes pattern-matched by
-   the function.  Plain lambdas are not included, e.g.
+      evaluates to `[1 2]`.
+    )",
+    .fun = prim_catAttrs,
+});
 
-      functionArgs (x: ...)
-   => { }
-*/
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
@@ -2131,7 +2142,7 @@ static RegisterPrimOp primop_functionArgs({
     .fun = prim_functionArgs,
 });
 
-/* Apply a function to every element of an attribute set. */
+/*  */
 static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[1], pos);
@@ -2147,6 +2158,21 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     }
 }
 
+static RegisterPrimOp primop_mapAttrs({
+    .name = "__mapAttrs",
+    .args = {"f", "attrset"},
+    .doc = R"(
+      Apply function *f* to every element of *attrset*. For example,
+
+      ```nix
+      builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }
+      ```
+
+      evaluates to `{ a = 10; b = 20; }`.
+    )",
+    .fun = prim_mapAttrs,
+});
+
 
 /*************************************************************
  * Lists
@@ -2582,9 +2608,29 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_partition({
+    .name = "__partition",
+    .args = {"pred", "list"},
+    .doc = R"(
+      Given a predicate function *pred*, this function returns an
+      attrset containing a list named `right`, containing the elements
+      in *list* for which *pred* returned `true`, and a list named
+      `wrong`, containing the elements for which it returned
+      `false`. For example,
+
+      ```nix
+      builtins.partition (x: x > 10) [1 23 9 3 42]
+      ```
+
+      evaluates to
+
+      ```nix
+      { right = [ 23 42 ]; wrong = [ 1 9 3 ]; }
+      ```
+    )",
+    .fun = prim_partition,
+});
 
-/* concatMap = f: list: concatLists (map f list); */
-/* C++-version is to avoid allocating `mkApp', call `f' eagerly */
 static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceFunction(*args[0], pos);
@@ -2611,6 +2657,16 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
     }
 }
 
+static RegisterPrimOp primop_concatMap({
+    .name = "__concatMap",
+    .args = {"f", "list"},
+    .doc = R"(
+      This function is equivalent to `builtins.concatLists (map f list)`
+      but is more efficient.
+    )",
+    .fun = prim_concatMap,
+});
+
 
 /*************************************************************
  * Integer arithmetic
@@ -3361,17 +3417,10 @@ void EvalState::createBaseEnv()
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
 
     // Paths
-    addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__findFile", 2, prim_findFile);
 
     // Sets
     addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos);
-    addPrimOp("__catAttrs", 2, prim_catAttrs);
-    addPrimOp("__mapAttrs", 2, prim_mapAttrs);
-
-    // Lists
-    addPrimOp("__partition", 2, prim_partition);
-    addPrimOp("__concatMap", 2, prim_concatMap);
 
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);

From 2a2121d264911abaa042be30238a14ec950c1ea7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Aug 2020 11:25:01 +0200
Subject: [PATCH 145/882] Use RegisterPrimOp for some undocumented primops

---
 src/libexpr/primops.cc | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index cdc526de3..2d3433200 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -156,7 +156,6 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     }
 }
 
-
 /* Want reasonable symbol names, so extern C */
 /* !!! Should we pass the Pos or the file name too? */
 extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
@@ -518,6 +517,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         v.listElems()[n++] = i;
 }
 
+static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
+    .name = "__genericClosure",
+    .arity = 1,
+    .fun = prim_genericClosure,
+});
 
 static RegisterPrimOp primop_abort({
     .name = "abort",
@@ -563,6 +567,11 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
     }
 }
 
+static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info {
+    .name = "__addErrorContext",
+    .arity = 2,
+    .fun = prim_addErrorContext,
+});
 
 /* Try evaluating the argument. Success => {success=true; value=something;},
  * else => {success=false; value=false;} */
@@ -1011,6 +1020,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     v.attrs->sort();
 }
 
+static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info {
+    .name = "derivationStrict",
+    .arity = 1,
+    .fun = prim_derivationStrict,
+});
 
 /* Return a placeholder string for the specified output that will be
    substituted by the corresponding output path at build time. For
@@ -1256,6 +1270,12 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
 }
 
+static RegisterPrimOp primop_findFile(RegisterPrimOp::Info {
+    .name = "__findFile",
+    .arity = 2,
+    .fun = prim_findFile,
+});
+
 /* Return the cryptographic hash of a file in base-16. */
 static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1901,6 +1921,12 @@ static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * *
         state.mkPos(v, i->pos);
 }
 
+static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info {
+    .name = "__unsafeGetAttrPos",
+    .arity = 2,
+    .fun = prim_unsafeGetAttrPos,
+});
+
 /* Dynamic version of the `?' operator. */
 static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -3413,17 +3439,6 @@ void EvalState::createBaseEnv()
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
     }
-    addPrimOp("__genericClosure", 1, prim_genericClosure);
-    addPrimOp("__addErrorContext", 2, prim_addErrorContext);
-
-    // Paths
-    addPrimOp("__findFile", 2, prim_findFile);
-
-    // Sets
-    addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos);
-
-    // Derivations
-    addPrimOp("derivationStrict", 1, prim_derivationStrict);
 
     /* Add a value containing the current Nix expression search path. */
     mkList(v, searchPath.size());

From f53b5f10585b1d4c939e45faf0dd2f4dacbca013 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Aug 2020 13:31:11 +0200
Subject: [PATCH 146/882] Add getDoc() function

---
 src/libexpr/eval.cc | 19 +++++++++++++++++++
 src/libexpr/eval.hh | 11 +++++++++++
 src/nix/repl.cc     | 25 ++++++++++++-------------
 3 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index a5ebd1bf0..4f2fd36c1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -563,6 +563,25 @@ Value & EvalState::getBuiltin(const string & name)
 }
 
 
+std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
+{
+    if (v.type == tPrimOp || v.type == tPrimOpApp) {
+        auto v2 = &v;
+        while (v2->type == tPrimOpApp)
+            v2 = v2->primOpApp.left;
+        if (v2->primOp->doc)
+            return Doc {
+                .pos = noPos,
+                .name = v2->primOp->name,
+                .arity = v2->primOp->arity,
+                .args = v2->primOp->args,
+                .doc = v2->primOp->doc,
+            };
+    }
+    return {};
+}
+
+
 /* Every "format" object (even temporary) takes up a few hundred bytes
    of stack space, which is a real killer in the recursive
    evaluator.  So here are some helper functions for throwing
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index db8eb3e16..80078d8a5 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -248,6 +248,17 @@ public:
 
     Value & getBuiltin(const string & name);
 
+    struct Doc
+    {
+        Pos pos;
+        std::optional<Symbol> name;
+        size_t arity;
+        std::vector<std::string> args;
+        const char * doc;
+    };
+
+    std::optional<Doc> getDoc(Value & v);
+
 private:
 
     inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index c7f8af742..8ecf11900 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -514,23 +514,22 @@ bool NixRepl::processLine(string line)
     else if (command == ":doc") {
         Value v;
         evalString(arg, v);
-        if (v.type == tPrimOp || v.type == tPrimOpApp) {
-            auto v2 = &v;
-            while (v2->type == tPrimOpApp)
-                v2 = v2->primOpApp.left;
-            if (v2->primOp->doc) {
-                auto args = v2->primOp->args;
+        if (auto doc = state->getDoc(v)) {
+            std::string markdown;
+
+            if (!doc->args.empty() && doc->name) {
+                auto args = doc->args;
                 for (auto & arg : args)
                     arg = "*" + arg + "*";
 
-                auto markdown =
-                    "**Synopsis:** `builtins." + (std::string) v2->primOp->name + "` "
-                    + concatStringsSep(" ", args) + "\n\n"
-                    + trim(stripIndentation(v2->primOp->doc));
+                markdown +=
+                    "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` "
+                    + concatStringsSep(" ", args) + "\n\n";
+            }
 
-                std::cout << renderMarkdownToTerminal(markdown);
-            } else
-                throw Error("builtin function '%s' does not have documentation", v2->primOp->name);
+            markdown += trim(stripIndentation(doc->doc));
+
+            std::cout << renderMarkdownToTerminal(markdown);
         } else
             throw Error("value does not have documentation");
     }

From 7a02865b94f107de1c1ef797b89d45ecd3f01cf1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Aug 2020 14:06:01 +0200
Subject: [PATCH 147/882] Move import docs

---
 doc/manual/src/expressions/builtins-prefix.md | 53 -----------
 src/libexpr/primops.cc                        | 93 ++++++++++++++++---
 tests/function-trace.sh                       | 12 ---
 3 files changed, 80 insertions(+), 78 deletions(-)

diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md
index ae3bb150c..0f7c3d32f 100644
--- a/doc/manual/src/expressions/builtins-prefix.md
+++ b/doc/manual/src/expressions/builtins-prefix.md
@@ -13,56 +13,3 @@ For instance, `derivation` is also available as `builtins.derivation`.
 
     `derivation` is described in [its own section](derivations.md).
 
-  - `import` *path*; `builtins.import` *path*  
-
-    Load, parse and return the Nix expression in the file *path*. If
-    *path* is a directory, the file ` default.nix ` in that directory
-    is loaded. Evaluation aborts if the file doesn’t exist or contains
-    an incorrect Nix expression. `import` implements Nix’s module
-    system: you can put any Nix expression (such as a set or a
-    function) in a separate file, and use it from Nix expressions in
-    other files.
-
-    > **Note**
-    >
-    > Unlike some languages, `import` is a regular function in Nix.
-    > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*)
-    > are [normal path values](language-values.md).
-
-    A Nix expression loaded by `import` must not contain any *free
-    variables* (identifiers that are not defined in the Nix expression
-    itself and are not built-in). Therefore, it cannot refer to
-    variables that are in scope at the call site. For instance, if you
-    have a calling expression
-
-    ```nix
-    rec {
-      x = 123;
-      y = import ./foo.nix;
-    }
-    ```
-
-    then the following `foo.nix` will give an error:
-
-    ```nix
-    x + 456
-    ```
-
-    since `x` is not in scope in `foo.nix`. If you want `x` to be
-    available in `foo.nix`, you should pass it as a function argument:
-
-    ```nix
-    rec {
-      x = 123;
-      y = import ./foo.nix x;
-    }
-    ```
-
-    and
-
-    ```nix
-    x: x + 456
-    ```
-
-    (The function argument doesn’t have to be called `x` in `foo.nix`;
-    any name would work.)
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 2d3433200..821cf0782 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -74,10 +74,10 @@ void EvalState::realiseContext(const PathSet & context)
 
 /* Load and evaluate an expression from path specified by the
    argument. */
-static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v)
 {
     PathSet context;
-    Path path = state.coerceToPath(pos, *args[1], context);
+    Path path = state.coerceToPath(pos, vPath, context);
 
     try {
         state.realiseContext(context);
@@ -99,6 +99,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             return std::nullopt;
         return storePath;
     };
+
     if (auto optStorePath = isValidDerivationInStore()) {
         auto storePath = *optStorePath;
         Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath));
@@ -133,17 +134,18 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         mkApp(v, **fun, w);
         state.forceAttrs(v, pos);
     } else {
-        state.forceAttrs(*args[0]);
-        if (args[0]->attrs->empty())
+        if (!vScope)
             state.evalFile(realPath, v);
         else {
-            Env * env = &state.allocEnv(args[0]->attrs->size());
+            state.forceAttrs(*vScope);
+
+            Env * env = &state.allocEnv(vScope->attrs->size());
             env->up = &state.baseEnv;
 
             StaticEnv staticEnv(false, &state.staticBaseEnv);
 
             unsigned int displ = 0;
-            for (auto & attr : *args[0]->attrs) {
+            for (auto & attr : *vScope->attrs) {
                 staticEnv.vars[attr.name] = displ;
                 env->values[displ++] = attr.value;
             }
@@ -156,6 +158,77 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     }
 }
 
+static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info {
+    .name = "scopedImport",
+    .arity = 2,
+    .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v)
+    {
+        import(state, pos, *args[1], args[0], v);
+    }
+});
+
+static RegisterPrimOp primop_import({
+    .name = "import",
+    .args = {"path"},
+    .doc = R"(
+      Load, parse and return the Nix expression in the file *path*. If
+      *path* is a directory, the file ` default.nix ` in that directory
+      is loaded. Evaluation aborts if the file doesn’t exist or contains
+      an incorrect Nix expression. `import` implements Nix’s module
+      system: you can put any Nix expression (such as a set or a
+      function) in a separate file, and use it from Nix expressions in
+      other files.
+
+      > **Note**
+      >
+      > Unlike some languages, `import` is a regular function in Nix.
+      > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*)
+      > are [normal path values](language-values.md).
+
+      A Nix expression loaded by `import` must not contain any *free
+      variables* (identifiers that are not defined in the Nix expression
+      itself and are not built-in). Therefore, it cannot refer to
+      variables that are in scope at the call site. For instance, if you
+      have a calling expression
+
+      ```nix
+      rec {
+        x = 123;
+        y = import ./foo.nix;
+      }
+      ```
+
+      then the following `foo.nix` will give an error:
+
+      ```nix
+      x + 456
+      ```
+
+      since `x` is not in scope in `foo.nix`. If you want `x` to be
+      available in `foo.nix`, you should pass it as a function argument:
+
+      ```nix
+      rec {
+        x = 123;
+        y = import ./foo.nix x;
+      }
+      ```
+
+      and
+
+      ```nix
+      x: x + 456
+      ```
+
+      (The function argument doesn’t have to be called `x` in `foo.nix`;
+      any name would work.)
+    )",
+    .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v)
+    {
+        import(state, pos, *args[0], nullptr, v);
+    }
+});
+
 /* Want reasonable symbol names, so extern C */
 /* !!! Should we pass the Pos or the file name too? */
 extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
@@ -3429,12 +3502,6 @@ void EvalState::createBaseEnv()
     addConstant("__langVersion", v);
 
     // Miscellaneous
-    auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport);
-    Value * v2 = allocValue();
-    mkAttrs(*v2, 0);
-    mkApp(v, *vScopedImport, *v2);
-    forceValue(v);
-    addConstant("import", v);
     if (evalSettings.enableNativeCode) {
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
@@ -3444,7 +3511,7 @@ void EvalState::createBaseEnv()
     mkList(v, searchPath.size());
     int n = 0;
     for (auto & i : searchPath) {
-        v2 = v.listElems()[n++] = allocValue();
+        auto v2 = v.listElems()[n++] = allocValue();
         mkAttrs(*v2, 2);
         mkString(*allocAttr(*v2, symbols.create("path")), i.second);
         mkString(*allocAttr(*v2, symbols.create("prefix")), i.first);
diff --git a/tests/function-trace.sh b/tests/function-trace.sh
index 182a4d5c2..3b7f364e3 100755
--- a/tests/function-trace.sh
+++ b/tests/function-trace.sh
@@ -32,8 +32,6 @@ expect_trace() {
 
 # failure inside a tryEval
 expect_trace 'builtins.tryEval (throw "example")' "
-function-trace entered undefined position at
-function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace entered (string):1:19 at
 function-trace exited (string):1:19 at
@@ -42,32 +40,24 @@ function-trace exited (string):1:1 at
 
 # Missing argument to a formal function
 expect_trace '({ x }: x) { }' "
-function-trace entered undefined position at
-function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace exited (string):1:1 at
 "
 
 # Too many arguments to a formal function
 expect_trace '({ x }: x) { x = "x"; y = "y"; }' "
-function-trace entered undefined position at
-function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace exited (string):1:1 at
 "
 
 # Not enough arguments to a lambda
 expect_trace '(x: y: x + y) 1' "
-function-trace entered undefined position at
-function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace exited (string):1:1 at
 "
 
 # Too many arguments to a lambda
 expect_trace '(x: x) 1 2' "
-function-trace entered undefined position at
-function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace exited (string):1:1 at
 function-trace entered (string):1:1 at
@@ -76,8 +66,6 @@ function-trace exited (string):1:1 at
 
 # Not a function
 expect_trace '1 2' "
-function-trace entered undefined position at
-function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace exited (string):1:1 at
 "

From c3efef9275ce6db3d65a2eb97aee507e79b626a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Aug 2020 09:28:10 +0200
Subject: [PATCH 148/882] Remove obsolete comment

---
 src/nix/local.mk | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/nix/local.mk b/src/nix/local.mk
index cfe97b0f0..e96200685 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,7 +15,6 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-# -fpermissive is needed by lowdown.
 nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil

From a0f19d9f3a009961869a077922f15986ce05738e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Aug 2020 14:48:08 +0200
Subject: [PATCH 149/882] RemoteStore::addToStore(): Fix race between
 stderrThread and NAR writer

As pointed out by @B4dM4n, the call to to.flush() on stderrThread is
unsafe because the NAR writer thread is also writing to 'to'.

Fixes #3943.
---
 src/libstore/remote-store.cc | 13 ++++++++-----
 src/libstore/remote-store.hh |  2 +-
 src/libutil/serialise.hh     |  6 ++++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index adba17c5e..e4a4ef5af 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -284,9 +284,9 @@ struct ConnectionHandle
 
     RemoteStore::Connection * operator -> () { return &*handle; }
 
-    void processStderr(Sink * sink = 0, Source * source = 0)
+    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
     {
-        auto ex = handle->processStderr(sink, source);
+        auto ex = handle->processStderr(sink, source, flush);
         if (ex) {
             daemonException = true;
             std::rethrow_exception(ex);
@@ -535,6 +535,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
 
+            conn->to.flush();
+
             std::exception_ptr ex;
 
             struct FramedSink : BufferedSink
@@ -574,7 +576,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             std::thread stderrThread([&]()
             {
                 try {
-                    conn.processStderr();
+                    conn.processStderr(nullptr, nullptr, false);
                 } catch (...) {
                     ex = std::current_exception();
                 }
@@ -884,9 +886,10 @@ static Logger::Fields readFields(Source & from)
 }
 
 
-std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
+std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
 {
-    to.flush();
+    if (flush)
+        to.flush();
 
     while (true) {
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index b319e774b..7cf4c4d12 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -114,7 +114,7 @@ protected:
 
         virtual ~Connection();
 
-        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
+        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
     };
 
     ref<Connection> openConnectionWrapper();
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 69ae0874a..8f17bc34c 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -23,7 +23,8 @@ struct Sink
 };
 
 
-/* A buffered abstract sink. */
+/* A buffered abstract sink. Warning: a BufferedSink should not be
+   used from multiple threads concurrently. */
 struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
@@ -66,7 +67,8 @@ struct Source
 };
 
 
-/* A buffered abstract source. */
+/* A buffered abstract source. Warning: a BufferedSink should not be
+   used from multiple threads concurrently. */
 struct BufferedSource : Source
 {
     size_t bufSize, bufPosIn, bufPosOut;

From e915fd6d2afb0299bcb77069503698faabe5f233 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Aug 2020 14:51:50 +0200
Subject: [PATCH 150/882] Typo

---
 src/libutil/serialise.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 8f17bc34c..a6f1c42e9 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -67,7 +67,7 @@ struct Source
 };
 
 
-/* A buffered abstract source. Warning: a BufferedSink should not be
+/* A buffered abstract source. Warning: a BufferedSource should not be
    used from multiple threads concurrently. */
 struct BufferedSource : Source
 {

From 626200713bb3cc844a9feb6af583c9b6b42c6dbc Mon Sep 17 00:00:00 2001
From: Griffin Smith <root@gws.fyi>
Date: Thu, 27 Aug 2020 12:28:12 -0400
Subject: [PATCH 151/882] Pass all args when auto-calling a function with an
 ellipsis

The command line options --arg and --argstr that are used by a bunch of
CLI commands to pass arguments to top-level functions in files go
through the same code-path as auto-calling top-level functions with
their default arguments - this, however, was only passing the arguments
that were *explicitly* mentioned in the formals of the function - in the
case of an as-pattern with an ellipsis (eg args @ { ... }) extra passed
arguments would get omitted. This fixes that to instead pass *all*
specified auto args in the case that our function has an ellipsis.

Fixes #598
---
 src/libexpr/eval.cc | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0123070d1..00191bce0 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1299,12 +1299,23 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     Value * actualArgs = allocValue();
     mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size());
 
-    for (auto & i : fun.lambda.fun->formals->formals) {
-        Bindings::iterator j = args.find(i.name);
-        if (j != args.end())
-            actualArgs->attrs->push_back(*j);
-        else if (!i.def)
-            throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+    if (fun.lambda.fun->formals->ellipsis) {
+        // If the formals have an ellipsis (eg the function accepts extra args) pass
+        // all available automatic arguments (which includes arguments specified on
+        // the command line via --arg/--argstr)
+        for (auto& v : args) {
+            actualArgs->attrs->push_back(v);
+        }
+    } else {
+        // Otherwise, only pass the arguments that the function accepts
+        for (auto & i : fun.lambda.fun->formals->formals) {
+            Bindings::iterator j = args.find(i.name);
+            if (j != args.end()) {
+                actualArgs->attrs->push_back(*j);
+            } else if (!i.def) {
+                throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+            }
+        }
     }
 
     actualArgs->attrs->sort();

From 3156560d413634eea170e8dd88b3c2208149d999 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Aug 2020 18:16:03 +0200
Subject: [PATCH 152/882] nix develop: Set output paths to writable locations

Currently, they're set to $(pwd)/outputs/$outputName. This allows
commands like 'make install' to work.
---
 flake.nix          | 11 ++-----
 src/nix/develop.cc | 81 +++++++++++++++++++++++++++++-----------------
 src/nix/get-env.sh | 17 +++++-----
 3 files changed, 63 insertions(+), 46 deletions(-)

diff --git a/flake.nix b/flake.nix
index a707e90e7..32ebb4936 100644
--- a/flake.nix
+++ b/flake.nix
@@ -421,6 +421,8 @@
         stdenv.mkDerivation {
           name = "nix";
 
+          outputs = [ "out" "dev" "doc" ];
+
           buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
 
           inherit configureFlags;
@@ -428,15 +430,6 @@
           enableParallelBuilding = true;
 
           installFlags = "sysconfdir=$(out)/etc";
-
-          shellHook =
-            ''
-              export prefix=$(pwd)/inst
-              configureFlags+=" --prefix=$prefix"
-              PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
-              PATH=$prefix/bin:$PATH
-              unset PYTHONPATH
-            '';
         });
 
   };
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 8a14e45c9..20316c477 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -15,7 +15,7 @@ struct Var
 {
     bool exported = true;
     bool associative = false;
-    std::string value; // quoted string or array
+    std::string quoted; // quoted string or array
 };
 
 struct BuildEnvironment
@@ -75,12 +75,12 @@ BuildEnvironment readEnvironment(const Path & path)
 
         else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
+            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
+            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
@@ -92,6 +92,8 @@ BuildEnvironment readEnvironment(const Path & path)
             path, file.substr(pos - file.cbegin(), 60));
     }
 
+    res.env.erase("__output");
+
     return res;
 }
 
@@ -125,27 +127,32 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs)
-        drv.env.erase(output.first);
-    drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
-    drv.env["out"] = "";
-    drv.env["_outputs_saved"] = drv.env["outputs"];
-    drv.env["outputs"] = "out";
+    for (auto & output : drv.outputs) {
+        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+        drv.env[output.first] = "";
+    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
-    auto shellOutPath = store->makeOutputPath("out", h, drv.name);
-    drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
-                .path = shellOutPath
-            } });
-    drv.env["out"] = store->printStorePath(shellOutPath);
-    auto shellDrvPath2 = writeDerivation(*store, drv);
+
+    for (auto & output : drv.outputs) {
+        auto outPath = store->makeOutputPath(output.first, h, drv.name);
+        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+        drv.env[output.first] = store->printStorePath(outPath);
+    }
+
+    auto shellDrvPath = writeDerivation(*store, drv);
 
     /* Build the derivation. */
-    store->buildPaths({{shellDrvPath2}});
+    store->buildPaths({{shellDrvPath}});
 
-    assert(store->isValidPath(shellOutPath));
+    for (auto & outPath : drv.outputPaths(*store)) {
+        assert(store->isValidPath(outPath));
+        auto outPathS = store->toRealPath(outPath);
+        if (lstat(outPathS).st_size)
+            return outPath;
+    }
 
-    return shellOutPath;
+    throw Error("get-env.sh failed to produce an environment");
 }
 
 struct Common : InstallableCommand, MixProfile
@@ -171,8 +178,12 @@ struct Common : InstallableCommand, MixProfile
         "UID",
     };
 
-    void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
+    std::string makeRcScript(
+        const BuildEnvironment & buildEnvironment,
+        const Path & outputsDir = absPath(".") + "/outputs")
     {
+        std::ostringstream out;
+
         out << "unset shellHook\n";
 
         out << "nix_saved_PATH=\"$PATH\"\n";
@@ -180,9 +191,9 @@ struct Common : InstallableCommand, MixProfile
         for (auto & i : buildEnvironment.env) {
             if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
                 if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.value);
+                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
                 else {
-                    out << fmt("%s=%s\n", i.first, i.second.value);
+                    out << fmt("%s=%s\n", i.first, i.second.quoted);
                     if (i.second.exported)
                         out << fmt("export %s\n", i.first);
                 }
@@ -193,13 +204,26 @@ struct Common : InstallableCommand, MixProfile
 
         out << buildEnvironment.bashFunctions << "\n";
 
-        // FIXME: set outputs
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
 
         out << "eval \"$shellHook\"\n";
+
+        /* Substitute occurrences of output paths. */
+        auto outputs = buildEnvironment.env.find("outputs");
+        assert(outputs != buildEnvironment.env.end());
+
+        // FIXME: properly unquote 'outputs'.
+        StringMap rewrites;
+        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
+            auto from = buildEnvironment.env.find(outputName);
+            assert(from != buildEnvironment.env.end());
+            // FIXME: unquote
+            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+        }
+
+        return rewriteStrings(out.str(), rewrites);
     }
 
     Strings getDefaultFlakeAttrPaths() override
@@ -288,19 +312,18 @@ struct CmdDevelop : Common, MixEnvironment
 
         auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
 
-        std::ostringstream ss;
-        makeRcScript(buildEnvironment, ss);
+        auto script = makeRcScript(buildEnvironment);
 
-        ss << fmt("rm -f '%s'\n", rcFilePath);
+        script +=  fmt("rm -f '%s'\n", rcFilePath);
 
         if (!command.empty()) {
             std::vector<std::string> args;
             for (auto s : command)
                 args.push_back(shellEscape(s));
-            ss << fmt("exec %s\n", concatStringsSep(" ", args));
+            script += fmt("exec %s\n", concatStringsSep(" ", args));
         }
 
-        writeFull(rcFileFd.get(), ss.str());
+        writeFull(rcFileFd.get(), script);
 
         stopProgressBar();
 
@@ -362,7 +385,7 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        makeRcScript(buildEnvironment, std::cout);
+        std::cout << makeRcScript(buildEnvironment);
     }
 };
 
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index 2e0e83561..091c0f573 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -1,12 +1,6 @@
 set -e
 if [ -e .attrs.sh ]; then source .attrs.sh; fi
 
-outputs=$_outputs_saved
-for __output in $_outputs_saved; do
-    declare "$__output"="$out"
-done
-unset _outputs_saved __output
-
 export IN_NIX_SHELL=impure
 export dontAddDisableDepTrack=1
 
@@ -14,5 +8,12 @@ if [[ -n $stdenv ]]; then
     source $stdenv/setup
 fi
 
-export > $out
-set >> $out
+for __output in $outputs; do
+    if [[ -z $__done ]]; then
+        export > ${!__output}
+        set >> ${!__output}
+        __done=1
+    else
+        echo -n >> ${!__output}
+    fi
+done

From 50a8710ed14c7a0236a656bb62d5117519ec5813 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Aug 2020 18:43:34 +0200
Subject: [PATCH 153/882] Close stdin while running tests

For some reason, the bash shell started by 'nix develop' sometimes
reads from stdin, which can hang.
---
 mk/tests.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mk/tests.mk b/mk/tests.mk
index 2e39bb694..c1e140bac 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -8,7 +8,7 @@ define run-install-test
 
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
+	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
 
 endef
 

From f15651303f8596bf34c67fc8d536b1e9e7843a87 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Aug 2020 19:24:29 +0200
Subject: [PATCH 154/882] nix develop: Add convenience flags for running
 specific phases

For example, for building the Nix flake, you would do:

  $ nix develop --configure
  $ nix develop --install
  $ nix develop --installcheck
---
 src/nix/develop.cc | 55 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 20316c477..516e7bda9 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -264,6 +264,7 @@ struct Common : InstallableCommand, MixProfile
 struct CmdDevelop : Common, MixEnvironment
 {
     std::vector<std::string> command;
+    std::optional<std::string> phase;
 
     CmdDevelop()
     {
@@ -277,6 +278,43 @@ struct CmdDevelop : Common, MixEnvironment
                 command = ss;
             }}
         });
+
+        addFlag({
+            .longName = "phase",
+            .description = "phase to run (e.g. `build` or `configure`)",
+            .labels = {"phase-name"},
+            .handler = {&phase},
+        });
+
+        addFlag({
+            .longName = "configure",
+            .description = "run the configure phase",
+            .handler = {&phase, {"configure"}},
+        });
+
+        addFlag({
+            .longName = "build",
+            .description = "run the build phase",
+            .handler = {&phase, {"build"}},
+        });
+
+        addFlag({
+            .longName = "check",
+            .description = "run the check phase",
+            .handler = {&phase, {"check"}},
+        });
+
+        addFlag({
+            .longName = "install",
+            .description = "run the install phase",
+            .handler = {&phase, {"install"}},
+        });
+
+        addFlag({
+            .longName = "installcheck",
+            .description = "run the installcheck phase",
+            .handler = {&phase, {"installCheck"}},
+        });
     }
 
     std::string description() override
@@ -314,9 +352,22 @@ struct CmdDevelop : Common, MixEnvironment
 
         auto script = makeRcScript(buildEnvironment);
 
-        script +=  fmt("rm -f '%s'\n", rcFilePath);
+        if (verbosity >= lvlDebug)
+            script += "set -x\n";
 
-        if (!command.empty()) {
+        script += fmt("rm -f '%s'\n", rcFilePath);
+
+        if (phase) {
+            if (!command.empty())
+                throw UsageError("you cannot use both '--command' and '--phase'");
+            // FIXME: foundMakefile is set by buildPhase, need to get
+            // rid of that.
+            script += fmt("foundMakefile=1\n");
+            script += fmt("runHook %1%Phase\n", *phase);
+            script += fmt("exit 0\n", *phase);
+        }
+
+        else if (!command.empty()) {
             std::vector<std::string> args;
             for (auto s : command)
                 args.push_back(shellEscape(s));

From 421ed527c70201c722cbefc10576ae77e383ba8e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 28 Aug 2020 17:22:57 -0400
Subject: [PATCH 155/882] Update src/libstore/build.cc

Thanks for catching, @regnat.
---
 src/libstore/build.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f5256bf87..1249668c4 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1477,8 +1477,7 @@ void DerivationGoal::inputsRealised()
                    });
 
             auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs,
-                buildMode == bmRepair ? bmRepair : bmNormal);
+                pathResolved, wantedOutputs, buildMode);
             addWaitee(resolvedGoal);
 
             state = &DerivationGoal::resolvedFinished;

From 4db0010a9374e357de3db3c0cf1cb1b490a14727 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 28 Aug 2020 22:03:54 +0000
Subject: [PATCH 156/882] Test CA derivation input caching

---
 tests/content-addressed.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 522310585..5997a432f 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -9,10 +9,13 @@ testDerivation () {
     local derivationPath=$1
     local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
     local out1=$(nix-build "${commonArgs[@]}" --arg seed 1)
-    local out2=$(nix-build "${commonArgs[@]}" --arg seed 2)
+    local out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${extraArgs[@]}")
     test $out1 == $out2
 }
 
 testDerivation root
+# The seed only changes the root derivation, and not it's output, so the
+# dependent derivations should only need to be built once.
+extaArgs=(-j0)
 testDerivation dependent
 testDerivation transitivelyDependent

From 8757e7022a9b407b69f173b3793fababf1e8ed84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Sun, 30 Aug 2020 19:41:21 +0200
Subject: [PATCH 157/882] mention how to run a single functional test for
 faster feedback loop

---
 doc/manual/hacking.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml
index d25d4b84a..267386d42 100644
--- a/doc/manual/hacking.xml
+++ b/doc/manual/hacking.xml
@@ -57,6 +57,12 @@ To install it in <literal>$(pwd)/inst</literal> and test it:
 nix (Nix) 2.4
 </screen>
 
+To run a functional test:
+          
+<screen>
+make tests/test-name-should-auto-complete.sh.test
+</screen>
+
 If you have a flakes-enabled nix you can replace:
 
 <screen>

From f38fe24346ca6b0201aed716db9cf47336a708ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Sun, 30 Aug 2020 22:52:34 +0200
Subject: [PATCH 158/882] speed up CI

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e16e6c62d..1f504a8ea 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,5 +13,7 @@ jobs:
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v10
+      with:
+        skip_adding_nixpkgs_channel: true
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

From ebb8e076eb41e712be2eff51fbde971ddddda1ee Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 31 Aug 2020 16:39:45 +0200
Subject: [PATCH 159/882] Restore some of the shellHook

---
 flake.nix | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/flake.nix b/flake.nix
index 32ebb4936..a0489ffd6 100644
--- a/flake.nix
+++ b/flake.nix
@@ -430,6 +430,12 @@
           enableParallelBuilding = true;
 
           installFlags = "sysconfdir=$(out)/etc";
+
+          shellHook =
+            ''
+              PATH=$prefix/bin:$PATH
+              unset PYTHONPATH
+            '';
         });
 
   };

From 6d7f7efb896c482ff6e7ce6c105e4dd3f7ee7218 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 1 Sep 2020 15:29:04 +0200
Subject: [PATCH 160/882] github: Use access token when calling .../commits API

---
 src/libfetchers/github.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 9f84ffb68..1cc0c5e2e 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -182,11 +182,21 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
+    void addAccessToken(std::string & url) const
+    {
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
+    }
+
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
         auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+
+        addAccessToken(url);
+
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
@@ -205,9 +215,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;
+        addAccessToken(url);
 
         return url;
     }

From a76b8b546753bc14aa2e8f8fdc74f70407aebd31 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Tue, 1 Sep 2020 12:05:49 -0700
Subject: [PATCH 161/882] README: update link to Hacking section

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 03c5deb7b..3cf4e44fa 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download
 
 ## Building And Developing
 
-See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to
+See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/hacking.html) in our manual for instruction on how to
 build nix from source with nix-build or how to get a development environment.
 
 ## Additional Resources

From a72ed3e8a1028c66c36b7dd66dc9dd50a0a6ecc1 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Tue, 1 Sep 2020 12:06:02 -0700
Subject: [PATCH 162/882] hacking.md: add --prefix flag to configure

Otherwise, the steps advertised in this document won't actually work
(e.g. `make install` will fail, trying to access /usr, and
`./inst/bin/nix` won't exist).
---
 doc/manual/src/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 9049e42bb..5bd884ce8 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -39,7 +39,7 @@ To build Nix itself in this shell:
 
 ```console
 [nix-shell]$ ./bootstrap.sh
-[nix-shell]$ ./configure $configureFlags
+[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/inst
 [nix-shell]$ make -j $NIX_BUILD_CORES
 ```
 

From dd4b56c87f8989c2d31a78b6c12cc2b1e3991fe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 1 Sep 2020 16:41:42 +0200
Subject: [PATCH 163/882] Allow HTTP binary cache to request absolute uris

---
 src/libstore/http-binary-cache-store.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index c1ceb08cf..1733239fb 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -85,7 +85,7 @@ protected:
         checkEnabled();
 
         try {
-            FileTransferRequest request(cacheUri + "/" + path);
+            FileTransferRequest request(makeRequest(path));
             request.head = true;
             getFileTransfer()->download(request);
             return true;
@@ -103,7 +103,7 @@ protected:
         std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
-        auto req = FileTransferRequest(cacheUri + "/" + path);
+        auto req = makeRequest(path);
         req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
         req.mimeType = mimeType;
         try {
@@ -115,8 +115,11 @@ protected:
 
     FileTransferRequest makeRequest(const std::string & path)
     {
-        FileTransferRequest request(cacheUri + "/" + path);
-        return request;
+        return FileTransferRequest(
+            hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
+            ? path
+            : cacheUri + "/" + path);
+
     }
 
     void getFile(const std::string & path, Sink & sink) override

From 94a043ff3b0e9de92bdb62372f9a82186d8e871c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Sep 2020 14:16:44 +0200
Subject: [PATCH 164/882] EvalCache: Fix caching of strings

This was broken in 50f13b06fb1b2f50a97323c000d1094d090f08ea. Once
again it turns out that putting a bool in a std::variant is a bad
idea, since pointers get silently cast to them...
---
 src/libexpr/eval-cache.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 46177a0a4..381344b40 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -391,7 +391,8 @@ Value & AttrCursor::forceValue()
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
         if (v.type == tString)
-            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
+            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
+                           string_t{v.string.s, {}}};
         else if (v.type == tPath)
             cachedValue = {root->db->setString(getKey(), v.path), v.path};
         else if (v.type == tBool)

From b74f5cdd2330ea1028eb6b04217bbe20c71a368b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Sep 2020 11:06:56 +0200
Subject: [PATCH 165/882] createGeneration(): Take a StorePath

---
 src/libstore/profiles.cc | 6 +++---
 src/libstore/profiles.hh | 4 +++-
 src/nix-env/nix-env.cc   | 4 +++-
 src/nix-env/user-env.cc  | 3 ++-
 src/nix/command.cc       | 2 +-
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 6862b42f0..e91dd781e 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -72,7 +72,7 @@ static void makeName(const Path & profile, GenerationNumber num,
 }
 
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
+Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
 {
     /* The new generation number should be higher than old the
        previous ones. */
@@ -82,7 +82,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
     if (gens.size() > 0) {
         Generation last = gens.back();
 
-        if (readLink(last.path) == outPath) {
+        if (readLink(last.path) == store->printStorePath(outPath)) {
             /* We only create a new generation symlink if it differs
                from the last one.
 
@@ -105,7 +105,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
        user environment etc. we've just built. */
     Path generation;
     makeName(profile, num + 1, generation);
-    store->addPermRoot(store->parseStorePath(outPath), generation, false, true);
+    store->addPermRoot(outPath, generation, false, true);
 
     return generation;
 }
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index abe507f0e..be55a65d4 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -8,6 +8,8 @@
 
 namespace nix {
 
+class StorePath;
+
 
 typedef unsigned int GenerationNumber;
 
@@ -28,7 +30,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
 
 class LocalFSStore;
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
+Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
 
 void deleteGeneration(const Path & profile, GenerationNumber gen);
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index ddd036070..e5a433ac0 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -708,7 +708,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     }
 
     debug(format("switching to new user environment"));
-    Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, drv.queryOutPath());
+    Path generation = createGeneration(
+        ref<LocalFSStore>(store2), globals.profile,
+        store2->parseStorePath(drv.queryOutPath()));
     switchLink(globals.profile, generation);
 }
 
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 8e7f09e12..8c6c8af05 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -151,7 +151,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
         }
 
         debug(format("switching to new user environment"));
-        Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut);
+        Path generation = createGeneration(ref<LocalFSStore>(store2), profile,
+            store2->parseStorePath(topLevelOut));
         switchLink(profile, generation);
     }
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 2f1fbded1..efac230bd 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -134,7 +134,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
     switchLink(profile2,
         createGeneration(
             ref<LocalFSStore>(store),
-            profile2, store->printStorePath(storePath)));
+            profile2, storePath));
 }
 
 void MixProfile::updateProfile(const Buildables & buildables)

From b07167be5aeb91c06ca3487b3d18f6c8f59942a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Sep 2020 11:12:38 +0200
Subject: [PATCH 166/882] createGeneration(): Always create an indirect root

This means profiles outside of /nix/var/nix/profiles don't get
garbage-collected. It also means we don't need to scan
/nix/var/nix/profiles for GC roots anymore, except for compatibility
with previously existing generations.
---
 src/libstore/profiles.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index e91dd781e..9b114f030 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -105,7 +105,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
        user environment etc. we've just built. */
     Path generation;
     makeName(profile, num + 1, generation);
-    store->addPermRoot(outPath, generation, false, true);
+    store->addPermRoot(outPath, generation, true);
 
     return generation;
 }

From 00d25e84577659ccf0bc360c61c47b6cd25d1c26 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Sep 2020 11:22:00 +0200
Subject: [PATCH 167/882] Remove the --indirect flag

All GC roots are now indirect.
---
 doc/manual/src/command-ref/nix-instantiate.md |  5 ++-
 doc/manual/src/command-ref/nix-store.md       | 34 ++++++-------------
 src/nix-instantiate/nix-instantiate.cc        |  7 ++--
 src/nix-store/nix-store.cc                    |  7 ++--
 tests/nix-build.sh                            |  2 +-
 tests/nix-shell.sh                            |  4 +--
 6 files changed, 21 insertions(+), 38 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 5b5ee0439..d09f5ed6a 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -12,7 +12,6 @@ Title: nix-instantiate
   [`--arg` *name* *value*]
   [{`--attr`| `-A`} *attrPath*]
   [`--add-root` *path*]
-  [`--indirect`]
   [`--expr` | `-E`]
   *files…*
 
@@ -32,8 +31,8 @@ standard input.
 
 # Options
 
-  - `--add-root` *path*; `--indirect`  
-    See the [corresponding options](nix-store.md) in `nix-store`.
+  - `--add-root` *path*  
+    See the [corresponding option](nix-store.md) in `nix-store`.
 
   - `--parse`  
     Just parse the input files, and print their abstract syntax trees on
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 193d670c2..4680339e4 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -9,7 +9,6 @@ Title: nix-store
 `nix-store` *operation* [*options…*] [*arguments…*]
   [`--option` *name* *value*]
   [`--add-root` *path*]
-  [`--indirect`]
 
 # Description
 
@@ -28,27 +27,12 @@ have an effect.
   - `--add-root` *path*  
     Causes the result of a realisation (`--realise` and
     `--force-realise`) to be registered as a root of the garbage
-    collector. The root is stored in *path*, which must be inside a
-    directory that is scanned for roots by the garbage collector
-    (i.e., typically in a subdirectory of `/nix/var/nix/gcroots/`)
-    *unless* the `--indirect` flag is used.
-
-    If there are multiple results, then multiple symlinks will be
-    created by sequentially numbering symlinks beyond the first one
-    (e.g., `foo`, `foo-2`, `foo-3`, and so on).
-
-  - `--indirect`  
-    In conjunction with `--add-root`, this option allows roots to be
-    stored *outside* of the GC roots directory. This is useful for
-    commands such as `nix-build` that place a symlink to the build
-    result in the current directory; such a build result should not be
-    garbage-collected unless the symlink is removed.
-
-    The `--indirect` flag causes a uniquely named symlink to *path* to
-    be stored in `/nix/var/nix/gcroots/auto/`. For instance,
+    collector. *path* will be created as a symlink to the resulting
+    store path. In addition, a uniquely named symlink to *path* will
+    be created in `/nix/var/nix/gcroots/auto/`. For instance,
 
     ```console
-    $ nix-store --add-root /home/eelco/bla/result --indirect -r ...
+    $ nix-store --add-root /home/eelco/bla/result -r ...
 
     $ ls -l /nix/var/nix/gcroots/auto
     lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
@@ -63,11 +47,13 @@ have an effect.
 
     > **Warning**
     >
-    > Note that it is not possible to move or rename indirect GC roots,
-    > since the symlink in the `auto` directory will still point to the
-    > old location.
+    > Note that it is not possible to move or rename GC roots, since
+    > the symlink in the `auto` directory will still point to the old
+    > location.
 
-<!-- end list -->
+    If there are multiple results, then multiple symlinks will be
+    created by sequentially numbering symlinks beyond the first one
+    (e.g., `foo`, `foo-2`, `foo-3`, and so on).
 
 # Operation `--realise`
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index bf353677a..d4996b93b 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -20,7 +20,6 @@ using namespace nix;
 
 static Path gcRoot;
 static int rootNr = 0;
-static bool indirectRoot = false;
 
 
 enum OutputKind { okPlain, okXML, okJSON };
@@ -71,11 +70,11 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 if (gcRoot == "")
                     printGCWarning();
                 else {
-                    Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot;
+                    Path rootName = absPath(gcRoot);
                     if (++rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
                     if (store2)
-                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot);
+                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, true);
                 }
                 std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : ""));
             }
@@ -127,7 +126,7 @@ static int _main(int argc, char * * argv)
             else if (*arg == "--add-root")
                 gcRoot = getArg(*arg, arg, end);
             else if (*arg == "--indirect")
-                indirectRoot = true;
+                ;
             else if (*arg == "--xml")
                 outputKind = okXML;
             else if (*arg == "--json")
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index a58edff57..4382b1460 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -34,7 +34,6 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs);
 
 static Path gcRoot;
 static int rootNr = 0;
-static bool indirectRoot = false;
 static bool noOutput = false;
 static std::shared_ptr<Store> store;
 
@@ -85,7 +84,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                     Path rootName = gcRoot;
                     if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     if (i->first != "out") rootName += "-" + i->first;
-                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot);
+                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, true);
                 }
             }
             outputs.insert(outPath);
@@ -104,7 +103,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                 Path rootName = gcRoot;
                 rootNr++;
                 if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
-                return {store2->addPermRoot(path.path, rootName, indirectRoot)};
+                return {store2->addPermRoot(path.path, rootName, true)};
             }
         }
         return {store->printStorePath(path.path)};
@@ -1085,7 +1084,7 @@ static int _main(int argc, char * * argv)
             else if (*arg == "--add-root")
                 gcRoot = absPath(getArg(*arg, arg, end));
             else if (*arg == "--indirect")
-                indirectRoot = true;
+                ;
             else if (*arg == "--no-output")
                 noOutput = true;
             else if (*arg != "" && arg->at(0) == '-') {
diff --git a/tests/nix-build.sh b/tests/nix-build.sh
index 0eb599608..3123c6da3 100644
--- a/tests/nix-build.sh
+++ b/tests/nix-build.sh
@@ -24,5 +24,5 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link)
 outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
 [[ $outPath = $outPath2 ]]
 
-outPath2=$(nix-store -r $(nix-instantiate --indirect --add-root $TEST_ROOT/indirect dependencies.nix)!out)
+outPath2=$(nix-store -r $(nix-instantiate --add-root $TEST_ROOT/indirect dependencies.nix)!out)
 [[ $outPath = $outPath2 ]]
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 650904057..1228bb04f 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -27,12 +27,12 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run
 # Test nix-shell on a .drv symlink
 
 # Legacy: absolute path and .drv extension required
-nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv
+nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell.drv
 [[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # New behaviour: just needs to resolve to a derivation in the store
-nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell
+nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell
 [[ $(nix-shell --pure $TEST_ROOT/shell --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 

From 82b77a77262c414044fffc7ad8b955ad91827995 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Sep 2020 11:26:36 +0200
Subject: [PATCH 168/882] addPermRoot(): Remove indirect flag

---
 src/libfetchers/registry.cc            |  2 +-
 src/libstore/gc.cc                     | 34 ++++++--------------------
 src/libstore/profiles.cc               |  2 +-
 src/libstore/store-api.hh              |  3 +--
 src/nix-build/nix-build.cc             |  2 +-
 src/nix-instantiate/nix-instantiate.cc |  2 +-
 src/nix-store/nix-store.cc             |  4 +--
 src/nix/build.cc                       |  4 +--
 src/nix/bundle.cc                      |  2 +-
 9 files changed, 17 insertions(+), 38 deletions(-)

diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index d4134ce29..4367ee810 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -147,7 +147,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
         if (!hasPrefix(path, "/")) {
             auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
+                store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
             path = store->toRealPath(storePath);
         }
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index e74382ed2..81f812604 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -85,8 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
 }
 
 
-Path LocalFSStore::addPermRoot(const StorePath & storePath,
-    const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
+Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
 {
     Path gcRoot(canonPath(_gcRoot));
 
@@ -95,31 +94,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
                 "(are you running nix-build inside the store?)", gcRoot);
 
-    if (indirect) {
-        /* Don't clobber the link if it already exists and doesn't
-           point to the Nix store. */
-        if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error("cannot create symlink '%1%'; already exists", gcRoot);
-        makeSymlink(gcRoot, printStorePath(storePath));
-        addIndirectRoot(gcRoot);
-    }
-
-    else {
-        if (!allowOutsideRootsDir) {
-            Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
-
-            if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
-                throw Error(
-                    "path '%1%' is not a valid garbage collector root; "
-                    "it's not in the directory '%2%'",
-                    gcRoot, rootsDir);
-        }
-
-        if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
-            writeFile(gcRoot, "");
-        else
-            makeSymlink(gcRoot, printStorePath(storePath));
-    }
+    /* Don't clobber the link if it already exists and doesn't
+       point to the Nix store. */
+    if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
+        throw Error("cannot create symlink '%1%'; already exists", gcRoot);
+    makeSymlink(gcRoot, printStorePath(storePath));
+    addIndirectRoot(gcRoot);
 
     /* Check that the root can be found by the garbage collector.
        !!! This can be very slow on machines that have many roots.
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 9b114f030..c20386e2b 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -105,7 +105,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
        user environment etc. we've just built. */
     Path generation;
     makeName(profile, num + 1, generation);
-    store->addPermRoot(outPath, generation, true);
+    store->addPermRoot(outPath, generation);
 
     return generation;
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0ec14b5f2..61aa3ba7e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -649,8 +649,7 @@ public:
     ref<FSAccessor> getFSAccessor() override;
 
     /* Register a permanent GC root. */
-    Path addPermRoot(const StorePath & storePath,
-        const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
+    Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
 
     virtual Path getRealStoreDir() { return storeDir; }
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 94412042f..471bcc10d 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -525,7 +525,7 @@ static void _main(int argc, char * * argv)
 
         for (auto & symlink : resultSymlinks)
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true);
+                store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first));
 
         logger->stop();
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index d4996b93b..539092cbe 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -74,7 +74,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                     if (++rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
                     if (store2)
-                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, true);
+                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName);
                 }
                 std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : ""));
             }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 4382b1460..3f2594712 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -84,7 +84,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                     Path rootName = gcRoot;
                     if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     if (i->first != "out") rootName += "-" + i->first;
-                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, true);
+                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName);
                 }
             }
             outputs.insert(outPath);
@@ -103,7 +103,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                 Path rootName = gcRoot;
                 rootNr++;
                 if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
-                return {store2->addPermRoot(path.path, rootName, true)};
+                return {store2->addPermRoot(path.path, rootName)};
             }
         }
         return {store->printStorePath(path.path)};
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 13d14a7fb..75a42ac55 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -71,14 +71,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
                         [&](BuildableOpaque bo) {
                             std::string symlink = outLink;
                             if (i) symlink += fmt("-%d", i);
-                            store2->addPermRoot(bo.path, absPath(symlink), true);
+                            store2->addPermRoot(bo.path, absPath(symlink));
                         },
                         [&](BuildableFromDrv bfd) {
                             for (auto & output : bfd.outputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);
-                                store2->addPermRoot(output.second, absPath(symlink), true);
+                                store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
                     }, buildables[i]);
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index eb3339f5d..241c8699b 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -122,7 +122,7 @@ struct CmdBundle : InstallableCommand
         if (!outLink)
             outLink = baseNameOf(app.program);
 
-        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink), true);
+        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink));
     }
 };
 

From 8a945d6ddb0676b454458e6fe0e9ea6f8b4b5659 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Sep 2020 11:30:15 +0200
Subject: [PATCH 169/882] Remove gc-check-reachability

---
 src/libstore/gc.cc      | 16 ----------------
 src/libstore/globals.hh |  4 ----
 2 files changed, 20 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 81f812604..e6cbc525d 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -101,22 +101,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
     makeSymlink(gcRoot, printStorePath(storePath));
     addIndirectRoot(gcRoot);
 
-    /* Check that the root can be found by the garbage collector.
-       !!! This can be very slow on machines that have many roots.
-       Instead of reading all the roots, it would be more efficient to
-       check if the root is in a directory in or linked from the
-       gcroots directory. */
-    if (settings.checkRootReachability) {
-        auto roots = findRoots(false);
-        if (roots[storePath].count(gcRoot) == 0)
-            logWarning({
-                .name = "GC root",
-                .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                    "therefore, '%2%' might be removed by the garbage collector",
-                    gcRoot, printStorePath(storePath))
-            });
-    }
-
     /* Grab the global GC root, causing us to block while a GC is in
        progress.  This prevents the set of permanent roots from
        increasing while a GC is in progress. */
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index ab9f42ce6..8a2d3ff75 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -293,10 +293,6 @@ public:
     Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
         "How often (in seconds) to poll for locks."};
 
-    Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
-        "Whether to check if new GC roots can in fact be found by the "
-        "garbage collector."};
-
     Setting<bool> gcKeepOutputs{
         this, false, "keep-outputs",
         R"(

From 145915eb3901ad78e61f27df22e755fdf9a2c28a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 3 Sep 2020 22:14:21 +0000
Subject: [PATCH 170/882] Beef up floating CA derivations test a bit

---
 tests/content-addressed.nix | 37 +++++++++++++++++++++++++------------
 tests/content-addressed.sh  |  7 ++++---
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 586e4cba6..1fa6aabff 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -4,16 +4,29 @@ with import ./config.nix;
 # A simple content-addressed derivation.
 # The derivation can be arbitrarily modified by passing a different `seed`,
 # but the output will always be the same
-mkDerivation {
-  name = "simple-content-addressed";
-  buildCommand = ''
-    set -x
-    echo "Building a CA derivation"
-    echo "The seed is ${toString seed}"
-    mkdir -p $out
-    echo "Hello World" > $out/hello
-  '';
-  __contentAddressed = true;
-  outputHashMode = "recursive";
-  outputHashAlgo = "sha256";
+rec {
+  rootLegacy = mkDerivation {
+    name = "simple-content-addressed";
+    buildCommand = ''
+      set -x
+      echo "Building a legacy derivation"
+      mkdir -p $out
+      echo "Hello World" > $out/hello
+    '';
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  };
+  rootCA = mkDerivation {
+    name = "dependent";
+    buildCommand = ''
+      echo "building a CA derivation"
+      echo "The seed is ${toString seed}"
+      mkdir -p $out
+      echo ${rootLegacy}/hello > $out/dep
+    '';
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  };
 }
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 2968f3a8c..ae9e3c59e 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -7,10 +7,11 @@ clearCache
 
 export REMOTE_STORE=file://$cacheDir
 
-drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1)
+drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1)
 nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
 
-out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link)
-out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link)
+commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "rootCA" "--no-out-link")
+out1=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 1)
+out2=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 2)
 
 test $out1 == $out2

From c224a5e5c1e0a55c8e8e7c3c1c0e50ac831f9964 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 3 Sep 2020 22:35:13 +0000
Subject: [PATCH 171/882] Rename derivation in floating CA test

---
 tests/content-addressed.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 1fa6aabff..79e8a8bf9 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -6,7 +6,7 @@ with import ./config.nix;
 # but the output will always be the same
 rec {
   rootLegacy = mkDerivation {
-    name = "simple-content-addressed";
+    name = "simple-input-addressed";
     buildCommand = ''
       set -x
       echo "Building a legacy derivation"

From aad4abcc9c27d5c1a2349e40f51f076387e0f844 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 01:17:38 +0000
Subject: [PATCH 172/882] Fix floating CA tests

We will sometimes try to query the outputs of derivations we can't
resolve. That's fine; it just means we don't know what those outputs are
yet.
---
 src/libstore/build.cc       | 4 +++-
 src/libstore/derivations.cc | 4 ++--
 src/libstore/derivations.hh | 2 +-
 src/libstore/local-store.cc | 9 +++++++--
 tests/content-addressed.sh  | 2 +-
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1249668c4..1f77b8ea8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1460,7 +1460,9 @@ void DerivationGoal::inputsRealised()
             /* We are be able to resolve this derivation based on the
                now-known results of dependencies. If so, we become a stub goal
                aliasing that resolved derivation goal */
-            Derivation drvResolved { fullDrv.resolve(worker.store) };
+            std::optional attempt = fullDrv.tryResolve(worker.store);
+            assert(attempt);
+            Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
             /* Add to memotable to speed up downstream goal's queries with the
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index ce57a5bb0..695265860 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -671,7 +671,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 Sync<DrvPathResolutions> drvPathResolutions;
 
-BasicDerivation Derivation::resolve(Store & store) {
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
     BasicDerivation resolved { *this };
 
     // Input paths that we'll want to rewrite in the derivation
@@ -683,7 +683,7 @@ BasicDerivation Derivation::resolve(Store & store) {
         for (auto & outputName : input.second) {
             auto actualPathOpt = inputDrvOutputs.at(outputName);
             if (!actualPathOpt)
-                throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first));
+                return std::nullopt;
             auto actualPath = *actualPathOpt;
             inputRewrites.emplace(
                 downstreamPlaceholder(store, input.first, outputName),
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index e4d85aa05..eaffbf452 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -133,7 +133,7 @@ struct Derivation : BasicDerivation
        1. input drv outputs moved to input sources.
 
        2. placeholders replaced with realized input store paths. */
-    BasicDerivation resolve(Store & store);
+    std::optional<BasicDerivation> tryResolve(Store & store);
 
     Derivation() = default;
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e51d127b3..f490188ce 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -825,8 +825,13 @@ std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivati
     /* can't just use else-if instead of `!haveCached` because we need to unlock
        `drvPathResolutions` before it is locked in `Derivation::resolve`. */
     if (!haveCached && drv.type() == DerivationType::CAFloating) {
-        /* Resolve drv and use that path instead. */
-        auto pathResolved = writeDerivation(*this, drv.resolve(*this));
+        /* Try resolve drv and use that path instead. */
+        auto attempt = drv.tryResolve(*this);
+        if (!attempt)
+            /* If we cannot resolve the derivation, we cannot have any path
+               assigned so we return the map of all std::nullopts. */
+            return outputs;
+        auto pathResolved = writeDerivation(*this, *std::move(attempt));
         /* Store in memo table. */
         /* FIXME: memo logic should not be local-store specific, should have
            wrapper-method instead. */
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index b2e94fe1e..34334b22d 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -9,7 +9,7 @@ testDerivation () {
     local derivationPath=$1
     local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
     local out1 out2
-    out1=$(set -e; nix-build "${commonArgs[@]}" --arg seed 1)
+    out1=$(nix-build "${commonArgs[@]}" --arg seed 1)
     out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${secondSeedArgs[@]}")
     test "$out1" == "$out2"
 }

From e12bcabdcbddc228d7af157bb3c2090e324c59a7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 02:30:12 +0000
Subject: [PATCH 173/882] Remove duplicate buildInputs

---
 flake.nix | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/flake.nix b/flake.nix
index 20c5089ee..843f9a85f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -69,6 +69,7 @@
             buildPackages.libxslt
             buildPackages.docbook5
             buildPackages.docbook_xsl_ns
+            buildPackages.autoconf-archive
             buildPackages.autoreconfHook
             buildPackages.pkgconfig
 
@@ -79,18 +80,9 @@
           ];
 
         buildDeps =
-          [ bison
-            flex
-            libxml2
-            libxslt
-            docbook5
-            docbook_xsl_ns
-            autoconf-archive
-            autoreconfHook
-
-            curl
+          [ curl
             bzip2 xz brotli zlib editline
-            openssl pkgconfig sqlite
+            openssl sqlite
             libarchive
             boost
             (if lib.versionAtLeast lib.version "20.03pre"
@@ -178,14 +170,17 @@
 
             src = self;
 
+            nativeBuildInputs =
+              [ buildPackages.autoconf-archive
+                buildPackages.autoreconfHook
+                buildPackages.pkgconfig
+              ];
+
             buildInputs =
-              [ autoconf-archive
-                autoreconfHook
-                nix
+              [ nix
                 curl
                 bzip2
                 xz
-                pkgconfig
                 pkgs.perl
                 boost
               ]

From ec14465a001387f8972c1b8332293d4fbce5ec97 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 02:43:56 +0000
Subject: [PATCH 174/882] Separate lowdown lib and bin to be more precise

---
 flake.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index 82eea55e2..9c79c0bbf 100644
--- a/flake.nix
+++ b/flake.nix
@@ -66,7 +66,7 @@
           [
             buildPackages.bison
             buildPackages.flex
-            buildPackages.lowdown
+            (lib.getBin buildPackages.lowdown)
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
@@ -208,7 +208,7 @@
 
           src = lowdown-src;
 
-          outputs = [ "out" "dev" ];
+          outputs = [ "out" "bin" "dev" ];
 
           nativeBuildInputs = [ which ];
 
@@ -216,7 +216,7 @@
             ''
               ./configure \
                 PREFIX=${placeholder "dev"} \
-                BINDIR=${placeholder "out"}/bin
+                BINDIR=${placeholder "bin"}/bin
             '';
         };
 

From b99062b023e56865ba20371b29fa3be6e6149d46 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 10:29:28 -0400
Subject: [PATCH 175/882] Update tests/content-addressed.nix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
---
 tests/content-addressed.nix | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 79e8a8bf9..b0eb29c3f 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -13,9 +13,6 @@ rec {
       mkdir -p $out
       echo "Hello World" > $out/hello
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
   rootCA = mkDerivation {
     name = "dependent";

From c9f1ed912c2ed3d53fdfe84cbb0012b5c7c2332f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 14:38:45 +0000
Subject: [PATCH 176/882] Don't chmod symlink before moving outputs around
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
---
 src/libstore/build.cc       | 11 ++++++++---
 tests/content-addressed.nix |  3 +++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ba28e78c8..f406c89d2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2065,7 +2065,7 @@ void linkOrCopy(const Path & from, const Path & to)
            file (e.g. 32000 of ext3), which is quite possible after a
            'nix-store --optimise'. FIXME: actually, why don't we just
            bind-mount in this case?
-           
+
            It can also fail with EPERM in BeegFS v7 and earlier versions
            which don't allow hard-links to other directories */
         if (errno != EMLINK && errno != EPERM)
@@ -4101,8 +4101,13 @@ void DerivationGoal::registerOutputs()
                     if (lstat(actualPath.c_str(), &st))
                         throw SysError("getting attributes of path '%1%'", actualPath);
                     mode |= 0200;
-                    if (chmod(actualPath.c_str(), mode) == -1)
-                        throw SysError("changing mode of '%1%' to %2$o", actualPath, mode);
+                    /* Try to change the perms, but only if the file isn't a
+                       symlink as symlinks permissions are mostly ignored and
+                       calling `chmod` on it will just forward the call to the
+                       target of the link. */
+                    if (!S_ISLNK(st.st_mode))
+                        if (chmod(actualPath.c_str(), mode) == -1)
+                            throw SysError("changing mode of '%1%' to %2$o", actualPath, mode);
                 }
                 if (rename(
                         actualPath.c_str(),
diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index b0eb29c3f..5e9bad0ac 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -16,11 +16,14 @@ rec {
   };
   rootCA = mkDerivation {
     name = "dependent";
+    outputs = [ "out" "dev" ];
     buildCommand = ''
       echo "building a CA derivation"
       echo "The seed is ${toString seed}"
       mkdir -p $out
       echo ${rootLegacy}/hello > $out/dep
+      # test symlink at root
+      ln -s $out $dev
     '';
     __contentAddressed = true;
     outputHashMode = "recursive";

From e86dd59dccb550c7f1a4d9333eee3c3a5c4043e9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 10:48:50 -0400
Subject: [PATCH 177/882] Apply suggestions from code review

Thanks!

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/build.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f406c89d2..5584249d2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1266,7 +1266,7 @@ void DerivationGoal::haveDerivation()
         for (auto & [_, status] : initialOutputs) {
             if (!status.wanted) continue;
             if (!status.known) {
-                warn("Do not know how to query for unknown floating CA drv output yet");
+                warn("do not know how to query for unknown floating content-addressed derivation output yet");
                 /* Nothing to wait for; tail call */
                 return DerivationGoal::gaveUpOnSubstitution();
             }
@@ -1463,7 +1463,7 @@ void DerivationGoal::inputsRealised()
                     auto optRealizedInput = outputs.at(j);
                     if (!optRealizedInput)
                         throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output.",
+                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
                             worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
                     worker.store.computeFSClosure(*optRealizedInput, inputPaths);
                 } else
@@ -2032,7 +2032,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
                        `computeFSClosure` on the output path, rather than
                        derivation itself. That doesn't seem right to me, so I
                        won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("export references including CA derivations (themselves) is not yet implemented");
+                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
                 worker.store.computeFSClosure(*k.second.second, paths);
             }
         }
@@ -2175,7 +2175,7 @@ void DerivationGoal::startBuilder()
            differ. */
         if (fixedFinalPath == scratchPath) continue;
 
-        /* Ensure scratch scratch path is ours to use */
+        /* Ensure scratch path is ours to use. */
         deletePath(worker.store.printStorePath(scratchPath));
 
         /* Rewrite and unrewrite paths */

From e9fad3006b0b6f3a6430fdbfc61392cac6834502 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 15:15:51 +0000
Subject: [PATCH 178/882] Fix some of the issues raised by @edolstra

 - More and better comments

 - The easier renames
---
 src/libstore/build.cc       | 25 +++++++++++++------------
 src/libstore/derivations.hh | 18 ++++++++++++++++++
 src/libstore/local-store.hh |  5 ++---
 src/nix/command.cc          |  5 +++--
 4 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5584249d2..9eff03f7b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -718,7 +718,7 @@ typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
 
-struct KnownInitialOutputStatus {
+struct InitialOutputStatus {
     StorePath path;
     /* The output optional indicates whether it's already valid; i.e. exists
        and is registered. If we're repairing, inner bool indicates whether the
@@ -731,9 +731,9 @@ struct KnownInitialOutputStatus {
     }
 };
 
-struct InitialOutputStatus {
+struct InitialOutput {
     bool wanted;
-    std::optional<KnownInitialOutputStatus> known;
+    std::optional<InitialOutputStatus> known;
 };
 
 class DerivationGoal : public Goal
@@ -770,7 +770,7 @@ private:
        immediate input paths). */
     StorePathSet inputPaths;
 
-    std::map<std::string, InitialOutputStatus> initialOutputs;
+    std::map<std::string, InitialOutput> initialOutputs;
 
     /* User selected for running the builder. */
     std::unique_ptr<UserLock> buildUser;
@@ -1050,11 +1050,14 @@ private:
     /* Forcibly kill the child process, if any. */
     void killChild();
 
-    /* Map a path to another (reproducably) so we can avoid overwriting outputs
+    /* Create alternative path calculated from but distinct from the
+       input, so we can avoid overwriting outputs (or other store paths)
        that already exist. */
     StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name alone, if one doesn't
-       want to use a random path for CA builds. */
+    /* Make a path to another based on the output name along with the
+       derivation hash. */
+    /* FIXME add option to randomize, so we can audit whether our
+       rewrites caught everything */
     StorePath makeFallbackPath(std::string_view outputName);
 
     void repairClosure();
@@ -2141,8 +2144,6 @@ void DerivationGoal::startBuilder()
            with the actual hashes. */
         auto scratchPath =
             !status.known
-                /* FIXME add option to randomize, so we can audit whether our
-                 * rewrites caught everything */
                 ? makeFallbackPath(outputName)
             : !needsHashRewrite()
                 /* Can always use original path in sandbox */
@@ -4582,19 +4583,19 @@ void DerivationGoal::checkPathValidity()
 {
     bool checkHash = buildMode == bmRepair;
     for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutputStatus status {
+        InitialOutput info {
             .wanted = wantOutput(i.first, wantedOutputs),
         };
         if (i.second) {
             auto outputPath = *i.second;
-            status.known = {
+            info.known = {
                 .path = outputPath,
                 .valid = !worker.store.isValidPath(outputPath)
                     ? std::optional<bool> {}
                     : !checkHash || worker.pathContentsGood(outputPath),
             };
         }
-        initialOutputs.insert_or_assign(i.first, status);
+        initialOutputs.insert_or_assign(i.first, info);
     }
 }
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index c5c6e90ca..2c0b1feab 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -145,6 +145,11 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
 // FIXME: remove
 bool isDerivation(const string & fileName);
 
+/* Calculate the name that will be used for the store path for this
+   output.
+
+   This is usually <drv-name>-<output-name>, but is just <drv-name> when
+   the output name is "out". */
 std::string outputPathName(std::string_view drvName, std::string_view outputName);
 
 // known CA drv's output hashes, current just for fixed-output derivations
@@ -194,8 +199,21 @@ struct Sink;
 Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
 
+/* This creates an opaque and almost certainly unique string
+   deterministically from the output name.
+
+   It is used as a placeholder to allow derivations to refer to their
+   own outputs without needing to use the hash of a derivation in
+   itself, making the hash near-impossible to calculate. */
 std::string hashPlaceholder(const std::string & outputName);
 
+/* This creates an opaque and almost certainly unique string
+   deterministically from a derivation path and output name.
+
+   It is used as a placeholder to allow derivations to refer to
+   content-addressed paths whose content --- and thus the path
+   themselves --- isn't yet known. This occurs when a derivation has a
+   dependency which is a CA derivation. */
 std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
 
 }
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index ea430fa14..d5e6d68ef 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -279,9 +279,8 @@ private:
        specified by the ‘secret-key-files’ option. */
     void signPathInfo(ValidPathInfo & info);
 
-    /* Add a mapping from the deriver of the path info (if specified) to its
-     * out path
-     */
+    /* Register the store path 'output' as the output named 'outputName' of
+       derivation 'deriver'. */
     void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
     void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index f697eb84c..37a4bc785 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -150,8 +150,9 @@ void MixProfile::updateProfile(const Buildables & buildables)
             },
             [&](BuildableFromDrv bfd) {
                 for (auto & output : bfd.outputs) {
-                    if (!output.second)
-                        throw Error("output path should be known because we just tried to build it");
+                    /* Output path should be known because we just tried to
+                       build it. */
+                    assert(!output.second);
                     result.push_back(*output.second);
                 }
             },

From 5aed6f9b25b8547feb67bbbfaa263d013b82fb52 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 15:58:42 +0000
Subject: [PATCH 179/882] Document `mkOutputString`

---
 src/libexpr/primops.cc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4e248f979..78dc314fa 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -75,6 +75,18 @@ void EvalState::realiseContext(const PathSet & context)
     }
 }
 
+/* Add and attribute to the given attribute map from the output name to
+   the output path, or a placeholder.
+
+   Where possible the path is used, but for floating CA derivations we
+   may not know it. For sake of determinism we always assume we don't
+   and instead put in a place holder. In either case, however, the
+   string context will contain the drv path and output name, so
+   downstream derivations will have the proper dependency, and in
+   addition, before building, the placeholder will be rewritten to be
+   the actual path.
+
+   The 'drv' and 'drvPath' outputs must correspond. */
 static void mkOutputString(EvalState & state, Value & v,
     const StorePath & drvPath, const BasicDerivation & drv,
     std::pair<string, DerivationOutput> o)

From 98dfd7531d6df6abc925a446f390c4a5bbb9a51d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 4 Sep 2020 18:33:58 +0000
Subject: [PATCH 180/882] Fix querying outputs for CA derivations some more

If we resolve using the known path of a derivation whose output we
didn't have, we previously blew up. Now we just fail gracefully,
returning the map of all outputs unknown.
---
 src/libstore/derivations.cc |  4 ++--
 src/libstore/derivations.hh |  4 +++-
 src/libstore/local-store.cc | 20 ++++++++++++++++----
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 695265860..afac00fc4 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -69,7 +69,7 @@ bool BasicDerivation::isBuiltin() const
 
 
 StorePath writeDerivation(Store & store,
-    const Derivation & drv, RepairFlag repair)
+    const Derivation & drv, RepairFlag repair, bool readOnly)
 {
     auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs)
@@ -79,7 +79,7 @@ StorePath writeDerivation(Store & store,
        held during a garbage collection). */
     auto suffix = std::string(drv.name) + drvExtension;
     auto contents = drv.unparse(store, false);
-    return settings.readOnlyMode
+    return readOnly || settings.readOnlyMode
         ? store.computeStorePathForText(suffix, contents, references)
         : store.addTextToStore(suffix, contents, references, repair);
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 8aa496143..716862127 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -146,7 +146,9 @@ enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(Store & store,
-    const Derivation & drv, RepairFlag repair = NoRepair);
+    const Derivation & drv,
+    RepairFlag repair = NoRepair,
+    bool readOnly = false);
 
 /* Read a derivation from a file. */
 Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index f490188ce..0755cfa91 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -728,7 +728,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
 {
     auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
     if (!use.next())
-        throw Error("path '%s' is not valid", printStorePath(path));
+        throw InvalidPath("path '%s' is not valid", printStorePath(path));
     return use.getInt(0);
 }
 
@@ -831,7 +831,8 @@ std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivati
             /* If we cannot resolve the derivation, we cannot have any path
                assigned so we return the map of all std::nullopts. */
             return outputs;
-        auto pathResolved = writeDerivation(*this, *std::move(attempt));
+        /* Just compute store path */
+        auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
         /* Store in memo table. */
         /* FIXME: memo logic should not be local-store specific, should have
            wrapper-method instead. */
@@ -841,8 +842,19 @@ std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivati
     return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
         auto state(_state.lock());
 
-        auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
-            (queryValidPathId(*state, path)));
+        uint64_t drvId;
+        try {
+            drvId = queryValidPathId(*state, path);
+        } catch (InvalidPath &) {
+            /* FIXME? if the derivation doesn't exist, we cannot have a mapping
+               for it. */
+            return outputs;
+        }
+
+        auto useQueryDerivationOutputs {
+            state->stmtQueryDerivationOutputs.use()
+            (drvId)
+        };
 
         while (useQueryDerivationOutputs.next())
             outputs.insert_or_assign(

From ee5906243ae37b7fe007db30aa575f46ae292a35 Mon Sep 17 00:00:00 2001
From: Gabriel Gonzalez <Gabriel439@gmail.com>
Date: Fri, 4 Sep 2020 20:05:43 -0700
Subject: [PATCH 181/882] Add `nix-shell` support for preserving PS1

Fixes https://github.com/NixOS/nix/issues/1268

`nix-shell` will now preserve `PS1` if the `NIX_SHELL_PRESERVE_PROMPT`
environment variable is set.
---
 src/nix-build/nix-build.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 471bcc10d..0400f94f5 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -98,8 +98,8 @@ static void _main(int argc, char * * argv)
 
     // List of environment variables kept for --pure
     std::set<string> keepVars{
-        "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM",
-        "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL",
+        "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL",
+        "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL",
         "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"
     };
 
@@ -446,7 +446,7 @@ static void _main(int argc, char * * argv)
                 "PATH=%4%:\"$PATH\"; "
                 "SHELL=%5%; "
                 "set +e; "
-                R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
+                R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "

From 8dbd57a6a5fe497bde9e647a3249c1ce0ea121ab Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Fri, 11 Sep 2020 19:21:40 +0200
Subject: [PATCH 182/882] Fix auto argument passing for more auto arguments
 than formals

The change in 626200713bb3cc844a9feb6af583c9b6b42c6dbc didn't account
for when the number of auto arguments is bigger than the number of
formal arguments. This causes the following:

  $ nix-instantiate --eval -E '{ ... }@args: args.foo' --argstr foo foo
  nix-instantiate: src/libexpr/attr-set.hh:55: void nix::Bindings::push_back(const nix::Attr&): Assertion `size_ < capacity_' failed.
  Aborted (core dumped)
---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 8c97b3760..dab11ce45 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1348,7 +1348,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     }
 
     Value * actualArgs = allocValue();
-    mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size());
+    mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
 
     if (fun.lambda.fun->formals->ellipsis) {
         // If the formals have an ellipsis (eg the function accepts extra args) pass

From 7cb5f643a6ee5b8ffee2e0bb9159a677ff76d17c Mon Sep 17 00:00:00 2001
From: Jade <software@lfcode.ca>
Date: Sat, 12 Sep 2020 13:08:40 -0700
Subject: [PATCH 183/882] docs+test: fix remaining installer downloads without
 -L (#4006)

Co-authored-by: lf- <lf-@users.noreply.github.com>
---
 doc/manual/src/release-notes/rl-1.7.md | 2 +-
 doc/manual/src/release-notes/rl-2.1.md | 2 +-
 tests/install-darwin.sh                | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-1.7.md b/doc/manual/src/release-notes/rl-1.7.md
index 8d49ae54e..fb18e797d 100644
--- a/doc/manual/src/release-notes/rl-1.7.md
+++ b/doc/manual/src/release-notes/rl-1.7.md
@@ -117,7 +117,7 @@ features:
   - The binary tarball installer has been improved. You can now install
     Nix by running:
     
-        $ bash <(curl https://nixos.org/nix/install)
+        $ bash <(curl -L https://nixos.org/nix/install)
 
   - More evaluation errors include position information. For instance,
     selecting a missing attribute will print something like
diff --git a/doc/manual/src/release-notes/rl-2.1.md b/doc/manual/src/release-notes/rl-2.1.md
index b88834c83..a9592657b 100644
--- a/doc/manual/src/release-notes/rl-2.1.md
+++ b/doc/manual/src/release-notes/rl-2.1.md
@@ -10,7 +10,7 @@ in certain situations. In addition, it has the following new features:
   - The Nix installer now supports performing a Multi-User
     installation for Linux computers which are running systemd. You
     can select a Multi-User installation by passing the `--daemon`
-    flag to the installer: `sh <(curl https://nixos.org/nix/install)
+    flag to the installer: `sh <(curl -L https://nixos.org/nix/install)
     --daemon`.
 
     The multi-user installer cannot handle systems with SELinux. If
diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index 9933eba94..7e44e54c4 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -53,7 +53,7 @@ trap finish EXIT
 
 # First setup Nix
 cleanup
-curl -o install https://nixos.org/nix/install
+curl -L -o install https://nixos.org/nix/install
 yes | bash ./install
 verify
 

From 525b38eee8fac48eb2a82fb78fa0a933a9eee2a4 Mon Sep 17 00:00:00 2001
From: aszlig <aszlig@nix.build>
Date: Sun, 13 Sep 2020 02:40:51 +0200
Subject: [PATCH 184/882] Fix unspecified behaviour in readStorePathCAMap

When deploying a Hydra instance with current Nix master, most builds
would not run because of errors like this:

  queue monitor: error: --- Error --- hydra-queue-runner
  error: --- UsageError --- nix-daemon
  not a content address because it is not in the form '<prefix>:<rest>': /nix/store/...-somedrv

The last error message is from parseContentAddress, which expects a
colon-separated string, however what we got here is a store path.

Looking at the worker protocol, the following message sent to the Nix
daemon caused the error above:

  0x1E -> wopQuerySubstitutablePathInfos
  0x01 -> Number of paths
  0x16 -> Length of string
  "/nix/store/...-somedrv"
  0x00 -> Length of string
  ""

Looking at writeStorePathCAMap, the store path is indeed the first field
that's transmitted. However, readStorePathCAMap expects it to be the
*second* field *on my machine*, since expression evaluation order is a
classic form of unspecified behaviour[1] in C++.

This has been introduced in https://github.com/NixOS/nix/pull/3689,
specifically in commit 66a62b3189c8c9b0965850e6b3c9b0fda0b50fd8.

[1]: https://en.wikipedia.org/wiki/Unspecified_behavior#Order_of_evaluation_of_subexpressions

Signed-off-by: aszlig <aszlig@nix.build>
---
 src/libstore/remote-store.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index e4a4ef5af..8bcf92301 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -43,8 +43,11 @@ StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
 {
     StorePathCAMap paths;
     auto count = readNum<size_t>(from);
-    while (count--)
-        paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
+    while (count--) {
+        auto path = store.parseStorePath(readString(from));
+        auto ca = parseContentAddressOpt(readString(from));
+        paths.insert_or_assign(path, ca);
+    }
     return paths;
 }
 

From c8b17212c8191d5a124c691957c05b971f6b4e5f Mon Sep 17 00:00:00 2001
From: Brian Leung <leungbk@mailfence.com>
Date: Sun, 13 Sep 2020 14:07:09 -0700
Subject: [PATCH 185/882] Add ccls files to .gitignore

---
 .gitignore | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitignore b/.gitignore
index 0ea27c8c8..573fa0005 100644
--- a/.gitignore
+++ b/.gitignore
@@ -126,4 +126,10 @@ GRTAGS
 GSYMS
 GTAGS
 
+# ccls
+/.ccls-cache
+
+# auto-generated compilation database
+compile_commands.json
+
 nix-rust/target

From a59e77d9e54e8e7bf0f3c3f40c22cd34b7a81225 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Sep 2020 13:48:51 +0200
Subject: [PATCH 186/882] nix-daemon: Lower verbosity of restricted setting
 warning

Fixes #3992.
---
 src/libstore/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index f35ddb522..8adabf549 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -232,7 +232,7 @@ struct ClientSettings
                 else if (setSubstituters(settings.extraSubstituters))
                     ;
                 else
-                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
+                    debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
             } catch (UsageError & e) {
                 warn(e.what());
             }

From 250f8a4bbae2aeafd3fa3e637b52fbbb59a7a8e0 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Sep 2020 17:19:25 +0200
Subject: [PATCH 187/882] Escape `${` in strings when printing Nix expressions

Otherwise the result of the printing can't be parsed back correctly by
Nix (because the unescaped `${` will be parsed as the begining of an
anti-quotation).

Fix #3989
---
 src/libexpr/eval.cc                 | 1 +
 tests/lang/eval-okay-ind-string.exp | 2 +-
 tests/user-envs.nix                 | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index dab11ce45..d678231c6 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -87,6 +87,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
             else if (*i == '\n') str << "\\n";
             else if (*i == '\r') str << "\\r";
             else if (*i == '\t') str << "\\t";
+            else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
             else str << *i;
         str << "\"";
         break;
diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/lang/eval-okay-ind-string.exp
index 9cf4bd2ee..7862331fa 100644
--- a/tests/lang/eval-okay-ind-string.exp
+++ b/tests/lang/eval-okay-ind-string.exp
@@ -1 +1 @@
-"This is an indented multi-line string\nliteral.  An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed.  Thus,\nin this case four spaces will be\nstripped from each line, even though\n  THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n  followed by a newline, it's stripped, but\n  that's not the case here. Two spaces are\n  stripped because of the \"  \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n  The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n   Tabs are not interpreted as whitespace (since we can't guess\n   what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n   space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored.  But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n  Similarly you can force an indentation level,\n  in this case to 2 spaces.  This works because the anti-quote\n  is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n  rm -f /var/run/opengl-driver\n  ln -sf 123 /var/run/opengl-driver\n\n  rm -f /var/log/slim.log\n   \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf  \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin         \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/          # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n"
+"This is an indented multi-line string\nliteral.  An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed.  Thus,\nin this case four spaces will be\nstripped from each line, even though\n  THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n  followed by a newline, it's stripped, but\n  that's not the case here. Two spaces are\n  stripped because of the \"  \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n  The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', \${.\n   Tabs are not interpreted as whitespace (since we can't guess\n   what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n   space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored.  But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n  Similarly you can force an indentation level,\n  in this case to 2 spaces.  This works because the anti-quote\n  is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n  rm -f /var/run/opengl-driver\n  ln -sf 123 /var/run/opengl-driver\n\n  rm -f /var/log/slim.log\n   \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf  \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin         \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/          # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n"
diff --git a/tests/user-envs.nix b/tests/user-envs.nix
index 1aa410cc9..43eff1a68 100644
--- a/tests/user-envs.nix
+++ b/tests/user-envs.nix
@@ -13,7 +13,7 @@ let
     builder = ./user-envs.builder.sh;
   } // {
     meta = {
-      description = "A silly test package";
+      description = "A silly test package with some \${escaped anti-quotation} in it";
     };
   });
 

From 057c6203b5337cb4833958f4a4f29f6587f4eb2f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 7 Sep 2020 11:26:09 +0200
Subject: [PATCH 188/882] gracefully handle old daemon versions

Add a fallback path in `queryPartialDerivationOutputMap` for daemons
that don't support it.

Also upstreams a couple methods from `SSHStore` to `RemoteStore` as this
is needed to handle the fallback path.
---
 src/libstore/remote-store.cc | 37 ++++++++++++++++++++++++++++++++----
 src/libstore/remote-store.hh | 10 ++++++++++
 src/libstore/ssh-store.cc    | 17 -----------------
 3 files changed, 43 insertions(+), 21 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8bcf92301..dc61951d3 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -1,5 +1,6 @@
 #include "serialise.hh"
 #include "util.hh"
+#include "remote-fs-accessor.hh"
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"
@@ -479,10 +480,26 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 
 std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
-    auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputMap << printStorePath(path);
-    conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+    if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
+        auto conn(getConnection());
+        conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+        conn.processStderr();
+        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+    } else {
+        // Fallback for old daemon versions.
+        // For floating-CA derivations (and their co-dependencies) this is an
+        // under-approximation as it only returns the paths that can be inferred
+        // from the derivation itself (and not the ones that are known because
+        // the have been built), but as old stores don't handle floating-CA
+        // derivations this shouldn't matter
+        auto derivation = readDerivation(path);
+        auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
+        std::map<std::string, std::optional<StorePath>> ret;
+        for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
+            ret.emplace(outputName, outputAndPath.second);
+        }
+        return ret;
+    }
 
 }
 
@@ -871,6 +888,18 @@ RemoteStore::Connection::~Connection()
     }
 }
 
+void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
+{
+    auto conn(connections->get());
+    conn->to << wopNarFromPath << printStorePath(path);
+    conn->processStderr();
+    copyNAR(conn->from, sink);
+}
+
+ref<FSAccessor> RemoteStore::getFSAccessor()
+{
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
+}
 
 static Logger::Fields readFields(Source & from)
 {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 7cf4c4d12..6f05f2197 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -131,6 +131,10 @@ protected:
 
     friend struct ConnectionHandle;
 
+    virtual ref<FSAccessor> getFSAccessor() override;
+
+    virtual void narFromPath(const StorePath & path, Sink & sink) override;
+
 private:
 
     std::atomic_bool failed{false};
@@ -149,6 +153,12 @@ public:
     bool sameMachine() override
     { return true; }
 
+    ref<FSAccessor> getFSAccessor() override
+    { return LocalFSStore::getFSAccessor(); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { LocalFSStore::narFromPath(path, sink); }
+
 private:
 
     ref<RemoteStore::Connection> openConnection() override;
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index caae6b596..6cb97c1f1 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -40,10 +40,6 @@ public:
     bool sameMachine() override
     { return false; }
 
-    void narFromPath(const StorePath & path, Sink & sink) override;
-
-    ref<FSAccessor> getFSAccessor() override;
-
 private:
 
     struct Connection : RemoteStore::Connection
@@ -68,19 +64,6 @@ private:
     };
 };
 
-void SSHStore::narFromPath(const StorePath & path, Sink & sink)
-{
-    auto conn(connections->get());
-    conn->to << wopNarFromPath << printStorePath(path);
-    conn->processStderr();
-    copyNAR(conn->from, sink);
-}
-
-ref<FSAccessor> SSHStore::getFSAccessor()
-{
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
-}
-
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();

From 733d2e9402807e54d503c3113e854bfddb3d44e0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 15 Sep 2020 13:48:23 +0200
Subject: [PATCH 189/882] .gitignore: inst -> outputs

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 573fa0005..3f81f8ef5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -107,7 +107,7 @@ perl/Makefile.config
 
 /src/resolve-system-dependencies/resolve-system-dependencies
 
-inst/
+outputs/
 
 *.a
 *.o

From c4bf219b55c76329988671d6b383d073373919f0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 15 Sep 2020 14:26:56 +0000
Subject: [PATCH 190/882] Don't link deriver until after any delayed exception
 is thrown

Otherwise, we will associate fixed-output derivations with outputs that
they did indeed produce, but which had the wrong hash. That's no good.
---
 src/libstore/build.cc | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9eff03f7b..d6a775ae9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4251,22 +4251,28 @@ void DerivationGoal::registerOutputs()
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            /* FIXME: we will want to track this mapping in the DB whether or
-               not we have a drv file. */
-            if (useDerivation)
-                worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
+    ValidPathInfos infos2;
+    for (auto & [outputName, newInfo] : infos) {
+        infos2.push_back(newInfo);
     }
+    worker.store.registerValidPaths(infos2);
 
     /* In case of a fixed-output derivation hash mismatch, throw an
        exception now that we have registered the output as valid. */
     if (delayedException)
         std::rethrow_exception(delayedException);
+
+    /* If we made it this far, we are sure the output matches the derivation
+       (since the delayedException would be a fixed output CA mismatch). That
+       means it's safe to link the derivation to the output hash. We must do
+       that for floating CA derivations, which otherwise couldn't be cached,
+       but it's fine to do in all cases. */
+    for (auto & [outputName, newInfo] : infos) {
+        /* FIXME: we will want to track this mapping in the DB whether or
+           not we have a drv file. */
+        if (useDerivation)
+            worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
+    }
 }
 
 

From 6387550d58884ac09204c339ed3a3cd700780a75 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 15 Sep 2020 15:19:45 +0000
Subject: [PATCH 191/882] Get rid of confusing `std::optional<bool>` for
 validity

---
 src/libstore/build.cc | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d6a775ae9..b800a432f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -718,16 +718,25 @@ typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
 
+/* Unless we are repairing, we don't both to test validity and just assume it,
+   so the choices are `Absent` or `Valid`. */
+enum struct PathStatus {
+    Corrupt,
+    Absent,
+    Valid,
+};
+
 struct InitialOutputStatus {
     StorePath path;
-    /* The output optional indicates whether it's already valid; i.e. exists
-       and is registered. If we're repairing, inner bool indicates whether the
-       valid path is in fact not corrupt. Otherwise, the inner bool is always
-       true (assumed no corruption). */
-    std::optional<bool> valid;
+    PathStatus status;
     /* Valid in the store, and additionally non-corrupt if we are repairing */
     bool isValid() const {
-        return valid && *valid;
+        return status == PathStatus::Valid;
+    }
+    /* Merely present, allowed to be corrupt */
+    bool isPresent() const {
+        return status == PathStatus::Corrupt
+            || status == PathStatus::Valid;
     }
 };
 
@@ -2148,10 +2157,10 @@ void DerivationGoal::startBuilder()
             : !needsHashRewrite()
                 /* Can always use original path in sandbox */
                 ? status.known->path
-            : !status.known->valid
+            : !status.known->isPresent()
                 /* If path doesn't yet exist can just use it */
                 ? status.known->path
-            : buildMode != bmRepair && !*status.known->valid
+            : buildMode != bmRepair && !status.known->isValid()
                 /* If we aren't repairing we'll delete a corrupted path, so we
                    can use original path */
                 ? status.known->path
@@ -2161,7 +2170,7 @@ void DerivationGoal::startBuilder()
         scratchOutputs.insert_or_assign(outputName, scratchPath);
 
         /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !*status.known->valid)
+        if (buildMode == bmRepair && !status.known->isValid())
             redirectedBadOutputs.insert(status.known->path);
 
         /* Substitute output placeholders with the scratch output paths.
@@ -4596,9 +4605,11 @@ void DerivationGoal::checkPathValidity()
             auto outputPath = *i.second;
             info.known = {
                 .path = outputPath,
-                .valid = !worker.store.isValidPath(outputPath)
-                    ? std::optional<bool> {}
-                    : !checkHash || worker.pathContentsGood(outputPath),
+                .status = !worker.store.isValidPath(outputPath)
+                    ? PathStatus::Absent
+                    : !checkHash || worker.pathContentsGood(outputPath)
+                    ? PathStatus::Valid
+                    : PathStatus::Corrupt,
             };
         }
         initialOutputs.insert_or_assign(i.first, info);

From 3a5cdd737cf529a7f641c4347f002b821a456a5d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 15 Sep 2020 15:21:39 +0000
Subject: [PATCH 192/882] Rename `Derivation::pathOpt` to `Derivation::path`

We no longer need the `*Opt` to disambiguate.
---
 src/libexpr/get-drvs.cc     | 2 +-
 src/libexpr/primops.cc      | 2 +-
 src/libstore/build.cc       | 4 ++--
 src/libstore/derivations.cc | 4 ++--
 src/libstore/derivations.hh | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index e66832182..91916e8bf 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -40,7 +40,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
         throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
     auto & [outputName, output] = *i;
 
-    auto optStorePath = output.pathOpt(*store, drv.name, outputName);
+    auto optStorePath = output.path(*store, drv.name, outputName);
     if (optStorePath)
         outPath = store->printStorePath(*optStorePath);
 }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 78dc314fa..4a0dd5544 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -91,7 +91,7 @@ static void mkOutputString(EvalState & state, Value & v,
     const StorePath & drvPath, const BasicDerivation & drv,
     std::pair<string, DerivationOutput> o)
 {
-    auto optOutputPath = o.second.pathOpt(*state.store, drv.name, o.first);
+    auto optOutputPath = o.second.path(*state.store, drv.name, o.first);
     mkString(
         *state.allocAttr(v, state.symbols.create(o.first)),
         optOutputPath
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b800a432f..ce973862d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4081,7 +4081,7 @@ void DerivationGoal::registerOutputs()
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         PathLocks dynamicOutputLock;
-        auto optFixedPath = output.pathOpt(worker.store, drv->name, outputName);
+        auto optFixedPath = output.path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)
         {
@@ -4574,7 +4574,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
     if (drv->type() != DerivationType::CAFloating) {
         std::map<std::string, std::optional<StorePath>> res;
         for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.pathOpt(worker.store, drv->name, name));
+            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
         return res;
     } else {
         return worker.store.queryPartialDerivationOutputMap(drvPath);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index e5ec60811..9d8ce5e36 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -7,7 +7,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName, std::string_view outputName) const
+std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
 {
     return std::visit(overloaded {
         [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
@@ -557,7 +557,7 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
     for (auto output : outputs)
         outsAndOptPaths.insert(std::make_pair(
             output.first,
-            std::make_pair(output.second, output.second.pathOpt(store, name, output.first))
+            std::make_pair(output.second, output.second.path(store, name, output.first))
             )
         );
     return outsAndOptPaths;
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 2c0b1feab..0b5652685 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -51,7 +51,7 @@ struct DerivationOutput
     /* Note, when you use this function you should make sure that you're passing
        the right derivation name. When in doubt, you should use the safer
        interface provided by BasicDerivation::outputsAndOptPaths */
-    std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName, std::string_view outputName) const;
+    std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;

From 7d5bdf8b5679cc7b2b9b4d9caf5af9ca52211336 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 8 Sep 2020 14:50:23 +0200
Subject: [PATCH 193/882] Make the store plugins more introspectable

Directly register the store classes rather than a function to build an
instance of them.
This gives the possibility to introspect static members of the class or
choose different ways of instantiating them.
---
 result                                   |  1 +
 src/libexpr/flake/lockfile.hh            |  2 +-
 src/libfetchers/fetchers.hh              |  2 +-
 src/libstore/dummy-store.cc              | 18 +++-----
 src/libstore/http-binary-cache-store.cc  | 24 +++++------
 src/libstore/legacy-ssh-store.cc         | 15 +++----
 src/libstore/local-binary-cache-store.cc | 23 +++++-----
 src/libstore/remote-store.cc             | 10 +----
 src/libstore/remote-store.hh             |  3 ++
 src/libstore/s3-binary-cache-store.cc    | 15 +++----
 src/libstore/ssh-store.cc                | 14 ++----
 src/libstore/store-api.cc                | 55 +++++++++++++-----------
 src/libstore/store-api.hh                | 31 ++++++++-----
 src/libstore/store.hh                    |  9 ++++
 src/nix/describe-stores.cc               | 30 +++++++++++++
 src/nix/develop.cc                       |  2 +-
 src/nix/diff-closures.cc                 |  2 +-
 src/nix/installables.hh                  |  2 +-
 18 files changed, 141 insertions(+), 117 deletions(-)
 create mode 120000 result
 create mode 100644 src/libstore/store.hh
 create mode 100644 src/nix/describe-stores.cc

diff --git a/result b/result
new file mode 120000
index 000000000..af7bbb6da
--- /dev/null
+++ b/result
@@ -0,0 +1 @@
+/nix/store/9pqfirjppd91mzhkgh8xnn66iwh53zk2-hello-2.10
\ No newline at end of file
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
index 5e7cfda3e..9ec8b39c3 100644
--- a/src/libexpr/flake/lockfile.hh
+++ b/src/libexpr/flake/lockfile.hh
@@ -6,7 +6,7 @@
 
 namespace nix {
 class Store;
-struct StorePath;
+class StorePath;
 }
 
 namespace nix::flake {
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index be71b786b..89b1e6e7d 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -23,7 +23,7 @@ struct InputScheme;
 
 struct Input
 {
-    friend class InputScheme;
+    friend struct InputScheme;
 
     std::shared_ptr<InputScheme> scheme; // note: can be null
     Attrs attrs;
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 7a5744bc1..a677e201e 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -2,17 +2,15 @@
 
 namespace nix {
 
-static std::string uriScheme = "dummy://";
-
 struct DummyStore : public Store
 {
-    DummyStore(const Params & params)
+    DummyStore(const std::string uri, const Params & params)
         : Store(params)
     { }
 
     string getUri() override
     {
-        return uriScheme;
+        return uriPrefixes()[0] + "://";
     }
 
     void queryPathInfoUncached(const StorePath & path,
@@ -21,6 +19,10 @@ struct DummyStore : public Store
         callback(nullptr);
     }
 
+    static std::vector<std::string> uriPrefixes() {
+        return {"dummy"};
+    }
+
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
@@ -48,12 +50,6 @@ struct DummyStore : public Store
     { unsupported("buildDerivation"); }
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (uri != uriScheme) return nullptr;
-    return std::make_shared<DummyStore>(params);
-});
+[[maybe_unused]] static RegisterStoreImplementation<DummyStore> regStore();
 
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 1733239fb..6c3a16faf 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -24,7 +24,8 @@ private:
 public:
 
     HttpBinaryCacheStore(
-        const Params & params, const Path & _cacheUri)
+        const Path & _cacheUri,
+        const Params & params)
         : BinaryCacheStore(params)
         , cacheUri(_cacheUri)
     {
@@ -55,6 +56,13 @@ public:
         }
     }
 
+    static std::vector<std::string> uriPrefixes()
+    {
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
+        auto ret = std::vector<std::string>({"http", "https"});
+        if (forceHttp) ret.push_back("file");
+        return ret;
+    }
 protected:
 
     void maybeDisable()
@@ -162,18 +170,6 @@ protected:
 
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
-    if (std::string(uri, 0, 7) != "http://" &&
-        std::string(uri, 0, 8) != "https://" &&
-        (!forceHttp || std::string(uri, 0, 7) != "file://"))
-        return 0;
-    auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
-    store->init();
-    return store;
-});
+[[maybe_unused]] static RegisterStoreImplementation<HttpBinaryCacheStore> regStore();
 
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index dc03313f0..72f28a82d 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -9,8 +9,6 @@
 
 namespace nix {
 
-static std::string uriScheme = "ssh://";
-
 struct LegacySSHStore : public Store
 {
     const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
@@ -37,6 +35,9 @@ struct LegacySSHStore : public Store
 
     SSHMaster master;
 
+    static std::vector<std::string> uriPrefixes() { return {"ssh"}; }
+
+
     LegacySSHStore(const string & host, const Params & params)
         : Store(params)
         , host(host)
@@ -84,7 +85,7 @@ struct LegacySSHStore : public Store
 
     string getUri() override
     {
-        return uriScheme + host;
+        return uriPrefixes()[0] + "://" + host;
     }
 
     void queryPathInfoUncached(const StorePath & path,
@@ -325,12 +326,6 @@ public:
     }
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
-});
+[[maybe_unused]] static RegisterStoreImplementation<LegacySSHStore> regStore();
 
 }
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 87d8334d7..752838d3f 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -13,7 +13,8 @@ private:
 public:
 
     LocalBinaryCacheStore(
-        const Params & params, const Path & binaryCacheDir)
+        const Path & binaryCacheDir,
+        const Params & params)
         : BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
@@ -26,6 +27,8 @@ public:
         return "file://" + binaryCacheDir;
     }
 
+    static std::vector<std::string> uriPrefixes();
+
 protected:
 
     bool fileExists(const std::string & path) override;
@@ -85,16 +88,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
+std::vector<std::string> LocalBinaryCacheStore::uriPrefixes()
 {
-    if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
-        std::string(uri, 0, 7) != "file://")
-        return 0;
-    auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
-    store->init();
-    return store;
-});
+    if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
+        return {};
+    else
+        return {"file"};
+}
+
+[[maybe_unused]] static RegisterStoreImplementation<LocalBinaryCacheStore> regStore();
 
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index dc61951d3..824b6b140 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -982,14 +982,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-static std::string uriScheme = "unix://";
-
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
-});
+[[maybe_unused]] static RegisterStoreImplementation<UDSRemoteStore> regStore();
 
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 6f05f2197..07cd5daf8 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -150,6 +150,9 @@ public:
 
     std::string getUri() override;
 
+    static std::vector<std::string> uriPrefixes()
+    { return {"unix"}; }
+
     bool sameMachine() override
     { return true; }
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index a0a446bd3..1bf4d5955 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -193,7 +193,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     S3Helper s3Helper;
 
     S3BinaryCacheStoreImpl(
-        const Params & params, const std::string & bucketName)
+        const std::string & bucketName,
+        const Params & params)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
@@ -426,17 +427,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return paths;
     }
 
+    static std::vector<std::string> uriPrefixes() { return {"s3"}; }
+
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, 5) != "s3://") return 0;
-    auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
-    store->init();
-    return store;
-});
+[[maybe_unused]] static RegisterStoreImplementation<S3BinaryCacheStoreImpl> regStore();
 
 }
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 6cb97c1f1..02208ee82 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -8,8 +8,6 @@
 
 namespace nix {
 
-static std::string uriScheme = "ssh-ng://";
-
 class SSHStore : public RemoteStore
 {
 public:
@@ -32,9 +30,11 @@ public:
     {
     }
 
+    static std::vector<std::string> uriPrefixes() { return {"ssh-ng"}; }
+
     std::string getUri() override
     {
-        return uriScheme + host;
+        return uriPrefixes()[0] + "://" + host;
     }
 
     bool sameMachine() override
@@ -76,12 +76,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
     return conn;
 }
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
-});
+[[maybe_unused]] static RegisterStoreImplementation<SSHStore> regStore();
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b3877487c..4a46b5160 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1009,6 +1009,11 @@ Derivation Store::readDerivation(const StorePath & drvPath)
     }
 }
 
+std::shared_ptr<Config> Store::getConfig()
+{
+    return shared_from_this();
+}
+
 
 }
 
@@ -1019,9 +1024,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
 
 namespace nix {
 
-
-RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
-
 /* Split URI into protocol+hierarchy part and its parameter set. */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
 {
@@ -1035,24 +1037,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
     return {uri, params};
 }
 
-ref<Store> openStore(const std::string & uri_,
-    const Store::Params & extraParams)
-{
-    auto [uri, uriParams] = splitUriAndParams(uri_);
-    auto params = extraParams;
-    params.insert(uriParams.begin(), uriParams.end());
-
-    for (auto fun : *RegisterStoreImplementation::implementations) {
-        auto store = fun(uri, params);
-        if (store) {
-            store->warnUnknownSettings();
-            return ref<Store>(store);
-        }
-    }
-
-    throw Error("don't know how to open Nix store '%s'", uri);
-}
-
 static bool isNonUriPath(const std::string & spec) {
     return
         // is not a URL
@@ -1080,10 +1064,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
     }
 }
 
-
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
+std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
 {
     switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
         case tDaemon:
@@ -1098,8 +1079,30 @@ static RegisterStoreImplementation regStore([](
         default:
             return nullptr;
     }
-});
+}
 
+ref<Store> openStore(const std::string & uri_,
+    const Store::Params & extraParams)
+{
+    auto [uri, uriParams] = splitUriAndParams(uri_);
+    auto params = extraParams;
+    params.insert(uriParams.begin(), uriParams.end());
+
+    if (auto store = openFromNonUri(uri, params)) {
+        store->warnUnknownSettings();
+        return ref<Store>(store);
+    }
+
+    for (auto implem : *implementations) {
+        auto store = implem.open(uri, params);
+        if (store) {
+            store->warnUnknownSettings();
+            return ref<Store>(store);
+        }
+    }
+
+    throw Error("don't know how to open Nix store '%s'", uri);
+}
 
 std::list<ref<Store>> getDefaultSubstituters()
 {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8b42aa6d2..28365542d 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -33,6 +33,7 @@ MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
 MakeError(BadStorePath, Error);
 
+MakeError(InvalidStoreURI, Error);
 
 class FSAccessor;
 class NarInfoDiskCache;
@@ -199,6 +200,8 @@ protected:
 
     Store(const Params & params);
 
+    std::shared_ptr<Config> getConfig();
+
 public:
 
     virtual ~Store() { }
@@ -744,25 +747,31 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
    ‘substituters’ option and various legacy options. */
 std::list<ref<Store>> getDefaultSubstituters();
 
+struct StoreFactory
+{
+    std::vector<std::string> uriPrefixes;
+    std::function<std::shared_ptr<Store> (const std::string & uri, const Store::Params & params)> open;
+};
+typedef std::vector<StoreFactory> Implementations;
+static Implementations * implementations = new Implementations;
 
-/* Store implementation registration. */
-typedef std::function<std::shared_ptr<Store>(
-    const std::string & uri, const Store::Params & params)> OpenStore;
-
+template<typename T>
 struct RegisterStoreImplementation
 {
-    typedef std::vector<OpenStore> Implementations;
-    static Implementations * implementations;
-
-    RegisterStoreImplementation(OpenStore fun)
+    RegisterStoreImplementation()
     {
-        if (!implementations) implementations = new Implementations;
-        implementations->push_back(fun);
+        StoreFactory factory{
+            .uriPrefixes = T::uriPrefixes(),
+            .open =
+                ([](const std::string & uri, const Store::Params & params)
+                 -> std::shared_ptr<Store>
+                 { return std::make_shared<T>(uri, params); })
+        };
+        implementations->push_back(factory);
     }
 };
 
 
-
 /* Display a set of paths in human-readable form (i.e., between quotes
    and separated by commas). */
 string showPaths(const PathSet & paths);
diff --git a/src/libstore/store.hh b/src/libstore/store.hh
new file mode 100644
index 000000000..bc73963f3
--- /dev/null
+++ b/src/libstore/store.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix {
+
+template<typename T> class BasicStore;
+class StoreConfig;
+typedef BasicStore<StoreConfig> Store;
+
+}
diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc
new file mode 100644
index 000000000..d84378316
--- /dev/null
+++ b/src/nix/describe-stores.cc
@@ -0,0 +1,30 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nix;
+
+struct CmdDescribeStores : Command, MixJSON
+{
+    std::string description() override
+    {
+        return "show registered store types and their available options";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+
+        if (json) {
+            auto availableStores = *implementations;
+        } else {
+            throw Error("Only json is available for describe-stores");
+        }
+    }
+};
+
+static auto r1 = registerCommand<CmdDescribeStores>("describe-stores");
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index a2ce9c8c1..f29fa71d2 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -392,7 +392,7 @@ struct CmdDevelop : Common, MixEnvironment
 
             auto bashInstallable = std::make_shared<InstallableFlake>(
                 state,
-                std::move(installable->nixpkgsFlakeRef()),
+                installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 lockFlags);
diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc
index 4199dae0f..0dc99d05e 100644
--- a/src/nix/diff-closures.cc
+++ b/src/nix/diff-closures.cc
@@ -81,7 +81,7 @@ void printClosureDiff(
         auto beforeSize = totalSize(beforeVersions);
         auto afterSize = totalSize(afterVersions);
         auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
-        auto showDelta = abs(sizeDelta) >= 8 * 1024;
+        auto showDelta = std::abs(sizeDelta) >= 8 * 1024;
 
         std::set<std::string> removed, unchanged;
         for (auto & [version, _] : beforeVersions)
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index 41c75a4ed..c7c2f8981 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -69,7 +69,7 @@ struct Installable
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
-        return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}));
+        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
 };
 

From fa32560169ffa55b9e0b6d5f04c3792acf0c544a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Sep 2020 11:18:12 +0200
Subject: [PATCH 194/882] Fix the registration of stores

---
 src/libstore/dummy-store.cc              |  2 +-
 src/libstore/http-binary-cache-store.cc  |  2 +-
 src/libstore/legacy-ssh-store.cc         |  2 +-
 src/libstore/local-binary-cache-store.cc |  2 +-
 src/libstore/remote-store.cc             |  2 +-
 src/libstore/s3-binary-cache-store.cc    |  2 +-
 src/libstore/ssh-store.cc                |  2 +-
 src/libstore/store-api.cc                |  3 ++-
 src/libstore/store-api.hh                | 29 +++++++++++++++---------
 9 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index a677e201e..dd1880877 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -50,6 +50,6 @@ struct DummyStore : public Store
     { unsupported("buildDerivation"); }
 };
 
-[[maybe_unused]] static RegisterStoreImplementation<DummyStore> regStore();
+static RegisterStoreImplementation<DummyStore> regStore;
 
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 6c3a16faf..c1abe35cb 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -170,6 +170,6 @@ protected:
 
 };
 
-[[maybe_unused]] static RegisterStoreImplementation<HttpBinaryCacheStore> regStore();
+static RegisterStoreImplementation<HttpBinaryCacheStore> regStore;
 
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 72f28a82d..552b4b176 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -326,6 +326,6 @@ public:
     }
 };
 
-[[maybe_unused]] static RegisterStoreImplementation<LegacySSHStore> regStore();
+static RegisterStoreImplementation<LegacySSHStore> regStore;
 
 }
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 752838d3f..808a1338f 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -96,6 +96,6 @@ std::vector<std::string> LocalBinaryCacheStore::uriPrefixes()
         return {"file"};
 }
 
-[[maybe_unused]] static RegisterStoreImplementation<LocalBinaryCacheStore> regStore();
+static RegisterStoreImplementation<LocalBinaryCacheStore> regStore;
 
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 824b6b140..5f455a62a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -982,6 +982,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-[[maybe_unused]] static RegisterStoreImplementation<UDSRemoteStore> regStore();
+static RegisterStoreImplementation<UDSRemoteStore> regStore;
 
 }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 1bf4d5955..f426b43a9 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -431,7 +431,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
 };
 
-[[maybe_unused]] static RegisterStoreImplementation<S3BinaryCacheStoreImpl> regStore();
+static RegisterStoreImplementation<S3BinaryCacheStoreImpl> regStore;
 
 }
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 02208ee82..8177a95b0 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -76,6 +76,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
     return conn;
 }
 
-[[maybe_unused]] static RegisterStoreImplementation<SSHStore> regStore();
+static RegisterStoreImplementation<SSHStore> regStore;
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 4a46b5160..0f321b434 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1093,7 +1093,7 @@ ref<Store> openStore(const std::string & uri_,
         return ref<Store>(store);
     }
 
-    for (auto implem : *implementations) {
+    for (auto implem : *Implementations::registered) {
         auto store = implem.open(uri, params);
         if (store) {
             store->warnUnknownSettings();
@@ -1136,5 +1136,6 @@ std::list<ref<Store>> getDefaultSubstituters()
     return stores;
 }
 
+std::vector<StoreFactory> * Implementations::registered = 0;
 
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 28365542d..61080978e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -749,25 +749,32 @@ std::list<ref<Store>> getDefaultSubstituters();
 
 struct StoreFactory
 {
-    std::vector<std::string> uriPrefixes;
     std::function<std::shared_ptr<Store> (const std::string & uri, const Store::Params & params)> open;
 };
-typedef std::vector<StoreFactory> Implementations;
-static Implementations * implementations = new Implementations;
+struct Implementations
+{
+    static std::vector<StoreFactory> * registered;
+
+    template<typename T>
+    static void add()
+    {
+        if (!registered) registered = new std::vector<StoreFactory>();
+        StoreFactory factory{
+            .open =
+                ([](const std::string & uri, const Store::Params & params)
+                 -> std::shared_ptr<Store>
+                 { return std::make_shared<T>(uri, params); }),
+        };
+        registered->push_back(factory);
+    }
+};
 
 template<typename T>
 struct RegisterStoreImplementation
 {
     RegisterStoreImplementation()
     {
-        StoreFactory factory{
-            .uriPrefixes = T::uriPrefixes(),
-            .open =
-                ([](const std::string & uri, const Store::Params & params)
-                 -> std::shared_ptr<Store>
-                 { return std::make_shared<T>(uri, params); })
-        };
-        implementations->push_back(factory);
+        Implementations::add<T>();
     }
 };
 

From 3b57181f8ed94cfa149ad4319ba96d41c5fbc30e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Sep 2020 11:29:17 +0200
Subject: [PATCH 195/882] Separate the instantiation and initialisation of the
 stores
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a new `init()` method to the `Store` class that is supposed to
handle all the effectful initialisation needed to set-up the store.
The constructor should remain side-effect free and just initialize the
c++ data structure.

The goal behind that is that we can create “dummy” instances of each
store to query static properties about it (the parameters it accepts for
example)
---
 src/libstore/binary-cache-store.hh       |  2 +-
 src/libstore/dummy-store.cc              |  4 ++++
 src/libstore/http-binary-cache-store.cc  |  6 ++++++
 src/libstore/legacy-ssh-store.cc         |  3 +++
 src/libstore/local-binary-cache-store.cc |  6 ++++++
 src/libstore/s3-binary-cache-store.cc    |  3 +++
 src/libstore/ssh-store.cc                |  6 ++++++
 src/libstore/store-api.cc                |  3 ++-
 src/libstore/store-api.hh                | 16 ++++++++++++----
 9 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 9bcdf5901..881398ea2 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -58,7 +58,7 @@ public:
 
 public:
 
-    virtual void init();
+    virtual void init() override;
 
 private:
 
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index dd1880877..52086445e 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -5,6 +5,10 @@ namespace nix {
 struct DummyStore : public Store
 {
     DummyStore(const std::string uri, const Params & params)
+        : DummyStore(params)
+    { }
+
+    DummyStore(const Params & params)
         : Store(params)
     { }
 
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index c1abe35cb..e76bac6e2 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -23,6 +23,12 @@ private:
 
 public:
 
+    HttpBinaryCacheStore(
+        const Params & params)
+        : HttpBinaryCacheStore("dummy", params)
+    {
+    }
+
     HttpBinaryCacheStore(
         const Path & _cacheUri,
         const Params & params)
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 552b4b176..777ba7520 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -37,6 +37,9 @@ struct LegacySSHStore : public Store
 
     static std::vector<std::string> uriPrefixes() { return {"ssh"}; }
 
+    LegacySSHStore(const Params & params)
+        : LegacySSHStore("dummy", params)
+    {}
 
     LegacySSHStore(const string & host, const Params & params)
         : Store(params)
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 808a1338f..bcd23eb82 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -12,6 +12,12 @@ private:
 
 public:
 
+    LocalBinaryCacheStore(
+        const Params & params)
+        : LocalBinaryCacheStore("dummy", params)
+    {
+    }
+
     LocalBinaryCacheStore(
         const Path & binaryCacheDir,
         const Params & params)
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f426b43a9..ab27f203f 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -192,6 +192,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     S3Helper s3Helper;
 
+    S3BinaryCacheStoreImpl(const Params & params)
+        : S3BinaryCacheStoreImpl("dummy-bucket", params) {}
+
     S3BinaryCacheStoreImpl(
         const std::string & bucketName,
         const Params & params)
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 8177a95b0..85ddce8be 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -17,6 +17,12 @@ public:
     const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
     const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
 
+    SSHStore(
+        const Params & params)
+        : SSHStore("dummy", params)
+    {
+    }
+
     SSHStore(const std::string & host, const Params & params)
         : Store(params)
         , RemoteStore(params)
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0f321b434..e14f361bd 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1094,8 +1094,9 @@ ref<Store> openStore(const std::string & uri_,
     }
 
     for (auto implem : *Implementations::registered) {
-        auto store = implem.open(uri, params);
+        auto store = implem.create(uri, params);
         if (store) {
+            store->init();
             store->warnUnknownSettings();
             return ref<Store>(store);
         }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 61080978e..6e081da41 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -200,9 +200,12 @@ protected:
 
     Store(const Params & params);
 
-    std::shared_ptr<Config> getConfig();
-
 public:
+    /**
+     * Perform any necessary effectful operation to make the store up and
+     * running
+     */
+    virtual void init() {};
 
     virtual ~Store() { }
 
@@ -749,7 +752,8 @@ std::list<ref<Store>> getDefaultSubstituters();
 
 struct StoreFactory
 {
-    std::function<std::shared_ptr<Store> (const std::string & uri, const Store::Params & params)> open;
+    std::function<std::shared_ptr<Store> (const std::string & uri, const Store::Params & params)> create;
+    std::function<std::shared_ptr<Store> (const Store::Params & params)> createDummy;
 };
 struct Implementations
 {
@@ -760,10 +764,14 @@ struct Implementations
     {
         if (!registered) registered = new std::vector<StoreFactory>();
         StoreFactory factory{
-            .open =
+            .create =
                 ([](const std::string & uri, const Store::Params & params)
                  -> std::shared_ptr<Store>
                  { return std::make_shared<T>(uri, params); }),
+            .createDummy =
+                ([](const Store::Params & params)
+                 -> std::shared_ptr<Store>
+                 { return std::make_shared<T>(params); })
         };
         registered->push_back(factory);
     }

From 3c525d15903ea98e5401ff57061295b8c625cc45 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Sep 2020 11:35:33 +0200
Subject: [PATCH 196/882] Complete the `toJSON` instance for `Setting<T>`

Don't let it just contain the value, but also the other fields of the
setting (description, aliases, etc..)
---
 src/libstore/globals.cc |  5 -----
 src/libutil/config.cc   | 32 ++++++++++----------------------
 src/libutil/config.hh   | 11 +++++++++--
 3 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 4a5971c3f..491c664db 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -162,11 +162,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
     else abort();
 }
 
-template<> nlohmann::json BaseSetting<SandboxMode>::toJSON()
-{
-    return AbstractSetting::toJSON();
-}
-
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
     args.addFlag({
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 3cf720bce..faa5cdbeb 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -137,11 +137,7 @@ nlohmann::json Config::toJSON()
     auto res = nlohmann::json::object();
     for (auto & s : _settings)
         if (!s.second.isAlias) {
-            auto obj = nlohmann::json::object();
-            obj.emplace("description", s.second.setting->description);
-            obj.emplace("aliases", s.second.setting->aliases);
-            obj.emplace("value", s.second.setting->toJSON());
-            res.emplace(s.first, obj);
+            res.emplace(s.first, s.second.setting->toJSON());
         }
     return res;
 }
@@ -168,19 +164,21 @@ void AbstractSetting::setDefault(const std::string & str)
 
 nlohmann::json AbstractSetting::toJSON()
 {
-    return to_string();
+    return nlohmann::json(toJSONObject());
+}
+
+std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
+{
+    std::map<std::string, nlohmann::json> obj;
+    obj.emplace("description", description);
+    obj.emplace("aliases", aliases);
+    return obj;
 }
 
 void AbstractSetting::convertToArg(Args & args, const std::string & category)
 {
 }
 
-template<typename T>
-nlohmann::json BaseSetting<T>::toJSON()
-{
-    return value;
-}
-
 template<typename T>
 void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
@@ -259,11 +257,6 @@ template<> std::string BaseSetting<Strings>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> nlohmann::json BaseSetting<Strings>::toJSON()
-{
-    return value;
-}
-
 template<> void BaseSetting<StringSet>::set(const std::string & str)
 {
     value = tokenizeString<StringSet>(str);
@@ -274,11 +267,6 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> nlohmann::json BaseSetting<StringSet>::toJSON()
-{
-    return value;
-}
-
 template class BaseSetting<int>;
 template class BaseSetting<unsigned int>;
 template class BaseSetting<long>;
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 2b4265806..be23076b0 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -206,7 +206,9 @@ protected:
 
     virtual std::string to_string() const = 0;
 
-    virtual nlohmann::json toJSON();
+    nlohmann::json toJSON();
+
+    virtual std::map<std::string, nlohmann::json> toJSONObject();
 
     virtual void convertToArg(Args & args, const std::string & category);
 
@@ -251,7 +253,12 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    nlohmann::json toJSON() override;
+    std::map<std::string, nlohmann::json> toJSONObject() override
+    {
+        auto obj = AbstractSetting::toJSONObject();
+        obj.emplace("value", value);
+        return obj;
+    }
 };
 
 template<typename T>

From 35042c96230e922b88c8a5cad6a7f5f06792860f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Sep 2020 11:36:48 +0200
Subject: [PATCH 197/882] Add a default value for the settings

The default value is initialized when creating the setting and unchanged
after that
---
 src/libutil/config.hh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index be23076b0..42fc856e0 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -222,6 +222,7 @@ class BaseSetting : public AbstractSetting
 protected:
 
     T value;
+    const T defaultValue;
 
 public:
 
@@ -231,6 +232,7 @@ public:
         const std::set<std::string> & aliases = {})
         : AbstractSetting(name, description, aliases)
         , value(def)
+        , defaultValue(def)
     { }
 
     operator const T &() const { return value; }
@@ -257,6 +259,7 @@ public:
     {
         auto obj = AbstractSetting::toJSONObject();
         obj.emplace("value", value);
+        obj.emplace("defaultValue", defaultValue);
         return obj;
     }
 };

From aa4eac37881258797c241113a5602913e1a5371a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Sep 2020 13:25:32 +0200
Subject: [PATCH 198/882] fixup! Separate the instantiation and initialisation
 of the stores

---
 src/libstore/store-api.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e14f361bd..831d4f30f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1009,12 +1009,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
     }
 }
 
-std::shared_ptr<Config> Store::getConfig()
-{
-    return shared_from_this();
-}
-
-
 }
 
 

From 22afa8fb4dd7e459faf531dd8a9c3580d02c7e2a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 10 Sep 2020 10:55:51 +0200
Subject: [PATCH 199/882] Separate store configs from the implems

Rework the `Store` hierarchy so that there's now one hierarchy for the
store configs and one for the implementations (where each implementation
extends the corresponding config). So a class hierarchy like

```
StoreConfig-------->Store
    |                 |
    v                 v
SubStoreConfig----->SubStore
    |                 |
    v                 v
SubSubStoreConfig-->SubSubStore
```

(with virtual inheritance to prevent DDD).

The advantage of this architecture is that we can now introspect the configuration of a store without having to instantiate the store itself
---
 src/libstore/binary-cache-store.cc       |  3 +-
 src/libstore/binary-cache-store.hh       |  8 +++-
 src/libstore/dummy-store.cc              |  4 +-
 src/libstore/http-binary-cache-store.cc  | 16 ++++----
 src/libstore/legacy-ssh-store.cc         | 17 ++++----
 src/libstore/local-binary-cache-store.cc | 18 ++++-----
 src/libstore/local-store.hh              |  2 +-
 src/libstore/remote-store.cc             | 10 ++---
 src/libstore/remote-store.hh             | 34 ++++++++++++----
 src/libstore/s3-binary-cache-store.cc    | 14 ++++---
 src/libstore/ssh-store.cc                | 25 ++++++------
 src/libstore/store-api.cc                |  2 +-
 src/libstore/store-api.hh                | 50 ++++++++++++++----------
 13 files changed, 120 insertions(+), 83 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 5433fe50d..34f844a18 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -22,7 +22,8 @@
 namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(const Params & params)
-    : Store(params)
+    : BinaryCacheStoreConfig(params)
+    , Store(params)
 {
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 881398ea2..f1fc82013 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -11,9 +11,9 @@ namespace nix {
 
 struct NarInfo;
 
-class BinaryCacheStore : public Store
+struct BinaryCacheStoreConfig : virtual StoreConfig
 {
-public:
+    using StoreConfig::StoreConfig;
 
     const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
@@ -22,6 +22,10 @@ public:
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{this, false, "parallel-compression",
         "enable multi-threading compression, available for xz only currently"};
+};
+
+class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
+{
 
 private:
 
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 52086445e..d4560c13d 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -2,6 +2,8 @@
 
 namespace nix {
 
+struct DummyStoreConfig : StoreConfig {};
+
 struct DummyStore : public Store
 {
     DummyStore(const std::string uri, const Params & params)
@@ -54,6 +56,6 @@ struct DummyStore : public Store
     { unsupported("buildDerivation"); }
 };
 
-static RegisterStoreImplementation<DummyStore> regStore;
+static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore;
 
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index e76bac6e2..c3fb455fa 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -7,7 +7,12 @@ namespace nix {
 
 MakeError(UploadToHTTP, Error);
 
-class HttpBinaryCacheStore : public BinaryCacheStore
+struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
+{
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+};
+
+class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
 {
 private:
 
@@ -23,16 +28,11 @@ private:
 
 public:
 
-    HttpBinaryCacheStore(
-        const Params & params)
-        : HttpBinaryCacheStore("dummy", params)
-    {
-    }
-
     HttpBinaryCacheStore(
         const Path & _cacheUri,
         const Params & params)
         : BinaryCacheStore(params)
+        , HttpBinaryCacheStoreConfig(params)
         , cacheUri(_cacheUri)
     {
         if (cacheUri.back() == '/')
@@ -176,6 +176,6 @@ protected:
 
 };
 
-static RegisterStoreImplementation<HttpBinaryCacheStore> regStore;
+static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore;
 
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 777ba7520..5a08fbba3 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -9,15 +9,21 @@
 
 namespace nix {
 
-struct LegacySSHStore : public Store
+struct LegacySSHStoreConfig : virtual StoreConfig
 {
+    using StoreConfig::StoreConfig;
     const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
     const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
     const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
+};
 
+struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
+{
     // Hack for getting remote build log output.
+    // Intentionally not in `StoreConfig` so that it doesn't appear in the
+    // documentation
     const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
 
     struct Connection
@@ -37,12 +43,9 @@ struct LegacySSHStore : public Store
 
     static std::vector<std::string> uriPrefixes() { return {"ssh"}; }
 
-    LegacySSHStore(const Params & params)
-        : LegacySSHStore("dummy", params)
-    {}
-
     LegacySSHStore(const string & host, const Params & params)
-        : Store(params)
+        : LegacySSHStoreConfig(params)
+        , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
@@ -329,6 +332,6 @@ public:
     }
 };
 
-static RegisterStoreImplementation<LegacySSHStore> regStore;
+static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore;
 
 }
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index bcd23eb82..fc3ae94c2 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -4,7 +4,12 @@
 
 namespace nix {
 
-class LocalBinaryCacheStore : public BinaryCacheStore
+struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
+{
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+};
+
+class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
 {
 private:
 
@@ -12,16 +17,11 @@ private:
 
 public:
 
-    LocalBinaryCacheStore(
-        const Params & params)
-        : LocalBinaryCacheStore("dummy", params)
-    {
-    }
-
     LocalBinaryCacheStore(
         const Path & binaryCacheDir,
         const Params & params)
-        : BinaryCacheStore(params)
+        : LocalBinaryCacheStoreConfig(params)
+        , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
     }
@@ -102,6 +102,6 @@ std::vector<std::string> LocalBinaryCacheStore::uriPrefixes()
         return {"file"};
 }
 
-static RegisterStoreImplementation<LocalBinaryCacheStore> regStore;
+static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore;
 
 }
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index d5e6d68ef..8b6972847 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -95,7 +95,7 @@ public:
 
 private:
 
-    Setting<bool> requireSigs{(Store*) this,
+    Setting<bool> requireSigs{(StoreConfig*) this,
         settings.requireSigs,
         "require-sigs", "whether store paths should have a trusted signature on import"};
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5f455a62a..619e9a7f1 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -94,6 +94,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : Store(params)
+    , RemoteStoreConfig(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
             [this]() { return openConnectionWrapper(); },
@@ -124,6 +125,7 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : Store(params)
+    , UDSRemoteStoreConfig(params)
     , LocalFSStore(params)
     , RemoteStore(params)
 {
@@ -131,11 +133,9 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
 
 
 UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
-    : Store(params)
-    , LocalFSStore(params)
-    , RemoteStore(params)
-    , path(socket_path)
+    : UDSRemoteStore(params)
 {
+    path.emplace(socket_path);
 }
 
 
@@ -982,6 +982,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-static RegisterStoreImplementation<UDSRemoteStore> regStore;
+static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
 
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 07cd5daf8..ba2f69427 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -16,19 +16,23 @@ struct FdSource;
 template<typename T> class Pool;
 struct ConnectionHandle;
 
+struct RemoteStoreConfig : virtual StoreConfig
+{
+    using StoreConfig::StoreConfig;
+
+    const Setting<int> maxConnections{(StoreConfig*) this, 1,
+            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
+
+    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
+            "max-connection-age", "number of seconds to reuse a connection"};
+};
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public virtual Store
+class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
 {
 public:
 
-    const Setting<int> maxConnections{(Store*) this, 1,
-            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
-
-    const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
-            "max-connection-age", "number of seconds to reuse a connection"};
-
     virtual bool sameMachine() = 0;
 
     RemoteStore(const Params & params);
@@ -141,7 +145,21 @@ private:
 
 };
 
-class UDSRemoteStore : public LocalFSStore, public RemoteStore
+struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
+{
+    UDSRemoteStoreConfig(const Store::Params & params)
+        : LocalFSStoreConfig(params)
+        , RemoteStoreConfig(params)
+    {
+    }
+
+    UDSRemoteStoreConfig()
+        : UDSRemoteStoreConfig(Store::Params({}))
+    {
+    }
+};
+
+class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
 {
 public:
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ab27f203f..b75d05555 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -172,8 +172,9 @@ S3Helper::FileTransferResult S3Helper::getObject(
     return res;
 }
 
-struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
+struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
     const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
     const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
@@ -185,20 +186,21 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
         this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
+};
 
+struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
+{
     std::string bucketName;
 
     Stats stats;
 
     S3Helper s3Helper;
 
-    S3BinaryCacheStoreImpl(const Params & params)
-        : S3BinaryCacheStoreImpl("dummy-bucket", params) {}
-
     S3BinaryCacheStoreImpl(
         const std::string & bucketName,
         const Params & params)
-        : S3BinaryCacheStore(params)
+        : S3BinaryCacheStoreConfig(params)
+        , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
     {
@@ -434,7 +436,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
 };
 
-static RegisterStoreImplementation<S3BinaryCacheStoreImpl> regStore;
+static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore;
 
 }
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 85ddce8be..2b6e2a298 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -8,23 +8,22 @@
 
 namespace nix {
 
-class SSHStore : public RemoteStore
+struct SSHStoreConfig : virtual RemoteStoreConfig
+{
+    using RemoteStoreConfig::RemoteStoreConfig;
+    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
+    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
+};
+
+class SSHStore : public RemoteStore, public virtual SSHStoreConfig
 {
 public:
 
-    const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
-    const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
-    const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
-
-    SSHStore(
-        const Params & params)
-        : SSHStore("dummy", params)
-    {
-    }
-
     SSHStore(const std::string & host, const Params & params)
         : Store(params)
+        , RemoteStoreConfig(params)
         , RemoteStore(params)
         , host(host)
         , master(
@@ -82,6 +81,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
     return conn;
 }
 
-static RegisterStoreImplementation<SSHStore> regStore;
+static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore;
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 831d4f30f..fe3f5221c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -346,7 +346,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
 
 
 Store::Store(const Params & params)
-    : Config(params)
+    : StoreConfig(params)
     , state({(size_t) pathInfoCacheSize})
 {
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6e081da41..61552f74e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -145,12 +145,9 @@ struct BuildResult
     }
 };
 
-
-class Store : public std::enable_shared_from_this<Store>, public Config
+struct StoreConfig : public Config
 {
-public:
-
-    typedef std::map<std::string, std::string> Params;
+    using Config::Config;
 
     const PathSetting storeDir_{this, false, settings.nixStore,
         "store", "path to the Nix store"};
@@ -168,6 +165,14 @@ public:
         "system-features",
         "Optional features that the system this store builds on implements (like \"kvm\")."};
 
+};
+
+class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
+{
+public:
+
+    typedef std::map<std::string, std::string> Params;
+
 protected:
 
     struct PathInfoCacheValue {
@@ -632,22 +637,25 @@ protected:
 
 };
 
-
-class LocalFSStore : public virtual Store
+struct LocalFSStoreConfig : virtual StoreConfig
 {
-public:
-
-    // FIXME: the (Store*) cast works around a bug in gcc that causes
+    using StoreConfig::StoreConfig;
+    // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
     // it to omit the call to the Setting constructor. Clang works fine
     // either way.
-    const PathSetting rootDir{(Store*) this, true, "",
+    const PathSetting rootDir{(StoreConfig*) this, true, "",
         "root", "directory prefixed to all other paths"};
-    const PathSetting stateDir{(Store*) this, false,
+    const PathSetting stateDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
         "state", "directory where Nix will store state"};
-    const PathSetting logDir{(Store*) this, false,
+    const PathSetting logDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
         "log", "directory where Nix will store state"};
+};
+
+class LocalFSStore : public virtual Store, public LocalFSStoreConfig
+{
+public:
 
     const static string drvsLogDir;
 
@@ -753,13 +761,13 @@ std::list<ref<Store>> getDefaultSubstituters();
 struct StoreFactory
 {
     std::function<std::shared_ptr<Store> (const std::string & uri, const Store::Params & params)> create;
-    std::function<std::shared_ptr<Store> (const Store::Params & params)> createDummy;
+    std::function<std::shared_ptr<StoreConfig> ()> getConfig;
 };
 struct Implementations
 {
     static std::vector<StoreFactory> * registered;
 
-    template<typename T>
+    template<typename T, typename TConfig>
     static void add()
     {
         if (!registered) registered = new std::vector<StoreFactory>();
@@ -768,21 +776,21 @@ struct Implementations
                 ([](const std::string & uri, const Store::Params & params)
                  -> std::shared_ptr<Store>
                  { return std::make_shared<T>(uri, params); }),
-            .createDummy =
-                ([](const Store::Params & params)
-                 -> std::shared_ptr<Store>
-                 { return std::make_shared<T>(params); })
+            .getConfig =
+                ([]()
+                 -> std::shared_ptr<StoreConfig>
+                 { return std::make_shared<TConfig>(); })
         };
         registered->push_back(factory);
     }
 };
 
-template<typename T>
+template<typename T, typename TConfig>
 struct RegisterStoreImplementation
 {
     RegisterStoreImplementation()
     {
-        Implementations::add<T>();
+        Implementations::add<T, TConfig>();
     }
 };
 

From dae39f0a7abfc41c122f4f2d54d6cf9e26cafe7a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 10 Sep 2020 11:03:36 +0200
Subject: [PATCH 200/882] Make `nix describe-stores` functional

Using the `*Config` class hierarchy
---
 src/nix/describe-stores.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc
index d84378316..f019ec9c2 100644
--- a/src/nix/describe-stores.cc
+++ b/src/nix/describe-stores.cc
@@ -20,7 +20,11 @@ struct CmdDescribeStores : Command, MixJSON
     {
 
         if (json) {
-            auto availableStores = *implementations;
+            auto res = nlohmann::json::array();
+            for (auto & implem : *Implementations::registered) {
+                auto storeConfig = implem.getConfig();
+                std::cout << storeConfig->toJSON().dump() << std::endl;
+            }
         } else {
             throw Error("Only json is available for describe-stores");
         }

From d184ad1d276006d4d003046710a56a019034a6a1 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Sep 2020 11:02:38 +0200
Subject: [PATCH 201/882] fixup! Make the store plugins more introspectable

---
 result | 1 -
 1 file changed, 1 deletion(-)
 delete mode 120000 result

diff --git a/result b/result
deleted file mode 120000
index af7bbb6da..000000000
--- a/result
+++ /dev/null
@@ -1 +0,0 @@
-/nix/store/9pqfirjppd91mzhkgh8xnn66iwh53zk2-hello-2.10
\ No newline at end of file

From 5895184df44e86ae55270390402c8263b0f24ae2 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Sep 2020 11:06:18 +0200
Subject: [PATCH 202/882] Correctly call all the parent contructors of the
 stores

Using virtual inheritance means that only the default constructors of
the parent classes will be called, which isn't what we want
---
 src/libstore/build.cc                    |  2 +-
 src/libstore/dummy-store.cc              | 13 +++++++++----
 src/libstore/http-binary-cache-store.cc  |  4 +++-
 src/libstore/legacy-ssh-store.cc         |  3 ++-
 src/libstore/local-binary-cache-store.cc |  4 +++-
 src/libstore/local-store.cc              |  5 ++++-
 src/libstore/local-store.hh              | 15 ++++++++++-----
 src/libstore/remote-store.cc             |  5 ++++-
 src/libstore/remote-store.hh             |  3 ++-
 src/libstore/s3-binary-cache-store.cc    |  4 +++-
 src/libstore/ssh-store.cc                |  7 +++++--
 src/libstore/store-api.hh                |  2 +-
 12 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ce973862d..4040a3fac 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2883,7 +2883,7 @@ struct RestrictedStore : public LocalFSStore
     DerivationGoal & goal;
 
     RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : Store(params), LocalFSStore(params), next(next), goal(goal)
+        : StoreConfig(params), LocalFSStoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
     { }
 
     Path getRealStoreDir() override
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index d4560c13d..68d03d934 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -2,17 +2,22 @@
 
 namespace nix {
 
-struct DummyStoreConfig : StoreConfig {};
+struct DummyStoreConfig : StoreConfig {
+    using StoreConfig::StoreConfig;
+};
 
-struct DummyStore : public Store
+struct DummyStore : public Store, public virtual DummyStoreConfig
 {
     DummyStore(const std::string uri, const Params & params)
         : DummyStore(params)
     { }
 
     DummyStore(const Params & params)
-        : Store(params)
-    { }
+        : StoreConfig(params)
+        , DummyStoreConfig(params)
+        , Store(params)
+    {
+    }
 
     string getUri() override
     {
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index c3fb455fa..2c9d61d51 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -31,7 +31,9 @@ public:
     HttpBinaryCacheStore(
         const Path & _cacheUri,
         const Params & params)
-        : BinaryCacheStore(params)
+        : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , BinaryCacheStore(params)
         , HttpBinaryCacheStoreConfig(params)
         , cacheUri(_cacheUri)
     {
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5a08fbba3..bbab452ab 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -44,7 +44,8 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
     static std::vector<std::string> uriPrefixes() { return {"ssh"}; }
 
     LegacySSHStore(const string & host, const Params & params)
-        : LegacySSHStoreConfig(params)
+        : StoreConfig(params)
+        , LegacySSHStoreConfig(params)
         , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index fc3ae94c2..becd74c75 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -20,7 +20,9 @@ public:
     LocalBinaryCacheStore(
         const Path & binaryCacheDir,
         const Params & params)
-        : LocalBinaryCacheStoreConfig(params)
+        : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , LocalBinaryCacheStoreConfig(params)
         , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0086bb13e..4e9840f3b 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -42,7 +42,10 @@ namespace nix {
 
 
 LocalStore::LocalStore(const Params & params)
-    : Store(params)
+    : StoreConfig(params)
+    , Store(params)
+    , LocalFSStoreConfig(params)
+    , LocalStoreConfig(params)
     , LocalFSStore(params)
     , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
         "physical path to the Nix store"}
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 8b6972847..0265f0837 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -30,8 +30,17 @@ struct OptimiseStats
     uint64_t blocksFreed = 0;
 };
 
+struct LocalStoreConfig : virtual LocalFSStoreConfig
+{
+    using LocalFSStoreConfig::LocalFSStoreConfig;
 
-class LocalStore : public LocalFSStore
+    Setting<bool> requireSigs{(StoreConfig*) this,
+        settings.requireSigs,
+        "require-sigs", "whether store paths should have a trusted signature on import"};
+};
+
+
+class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
 {
 private:
 
@@ -95,10 +104,6 @@ public:
 
 private:
 
-    Setting<bool> requireSigs{(StoreConfig*) this,
-        settings.requireSigs,
-        "require-sigs", "whether store paths should have a trusted signature on import"};
-
     const PublicKeys & getPublicKeys();
 
 public:
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 619e9a7f1..9bbb69b99 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -124,7 +124,10 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 
 
 UDSRemoteStore::UDSRemoteStore(const Params & params)
-    : Store(params)
+    : StoreConfig(params)
+    , Store(params)
+    , LocalFSStoreConfig(params)
+    , RemoteStoreConfig(params)
     , UDSRemoteStoreConfig(params)
     , LocalFSStore(params)
     , RemoteStore(params)
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index ba2f69427..32daab59a 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -148,7 +148,8 @@ private:
 struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
 {
     UDSRemoteStoreConfig(const Store::Params & params)
-        : LocalFSStoreConfig(params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
         , RemoteStoreConfig(params)
     {
     }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index b75d05555..30ef4082a 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -199,7 +199,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
     S3BinaryCacheStoreImpl(
         const std::string & bucketName,
         const Params & params)
-        : S3BinaryCacheStoreConfig(params)
+        : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , S3BinaryCacheStoreConfig(params)
         , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 2b6e2a298..da201a9c3 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -11,20 +11,23 @@ namespace nix {
 struct SSHStoreConfig : virtual RemoteStoreConfig
 {
     using RemoteStoreConfig::RemoteStoreConfig;
+
     const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
     const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
 };
 
-class SSHStore : public RemoteStore, public virtual SSHStoreConfig
+class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
 {
 public:
 
     SSHStore(const std::string & host, const Params & params)
-        : Store(params)
+        : StoreConfig(params)
+        , Store(params)
         , RemoteStoreConfig(params)
         , RemoteStore(params)
+        , SSHStoreConfig(params)
         , host(host)
         , master(
             host,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 61552f74e..8e9cb786c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -653,7 +653,7 @@ struct LocalFSStoreConfig : virtual StoreConfig
         "log", "directory where Nix will store state"};
 };
 
-class LocalFSStore : public virtual Store, public LocalFSStoreConfig
+class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
 {
 public:
 

From 7f103dcddd4cf3c0748b7a379a117f3262c4c5f7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Sep 2020 11:11:05 +0200
Subject: [PATCH 203/882] Properly filter the stores according to their
 declared uriSchemes

When opening a store, only try the stores whose `uriSchemes()` include
the current one
---
 src/libstore/dummy-store.cc              |  6 ++---
 src/libstore/http-binary-cache-store.cc  |  9 +++----
 src/libstore/legacy-ssh-store.cc         |  6 ++---
 src/libstore/local-binary-cache-store.cc |  5 ++--
 src/libstore/remote-store.cc             |  5 +++-
 src/libstore/remote-store.hh             |  4 ++--
 src/libstore/s3-binary-cache-store.cc    |  3 ++-
 src/libstore/ssh-store.cc                |  6 ++---
 src/libstore/store-api.cc                | 30 ++++++++++++++++--------
 src/libstore/store-api.hh                | 10 ++++----
 10 files changed, 51 insertions(+), 33 deletions(-)

diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 68d03d934..210e5b11c 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -8,7 +8,7 @@ struct DummyStoreConfig : StoreConfig {
 
 struct DummyStore : public Store, public virtual DummyStoreConfig
 {
-    DummyStore(const std::string uri, const Params & params)
+    DummyStore(const std::string scheme, const std::string uri, const Params & params)
         : DummyStore(params)
     { }
 
@@ -21,7 +21,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
 
     string getUri() override
     {
-        return uriPrefixes()[0] + "://";
+        return *uriSchemes().begin();
     }
 
     void queryPathInfoUncached(const StorePath & path,
@@ -30,7 +30,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
         callback(nullptr);
     }
 
-    static std::vector<std::string> uriPrefixes() {
+    static std::set<std::string> uriSchemes() {
         return {"dummy"};
     }
 
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 2c9d61d51..095eaa82c 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -29,13 +29,14 @@ private:
 public:
 
     HttpBinaryCacheStore(
+        const std::string & scheme,
         const Path & _cacheUri,
         const Params & params)
         : StoreConfig(params)
         , BinaryCacheStoreConfig(params)
         , BinaryCacheStore(params)
         , HttpBinaryCacheStoreConfig(params)
-        , cacheUri(_cacheUri)
+        , cacheUri(scheme + "://" + _cacheUri)
     {
         if (cacheUri.back() == '/')
             cacheUri.pop_back();
@@ -64,11 +65,11 @@ public:
         }
     }
 
-    static std::vector<std::string> uriPrefixes()
+    static std::set<std::string> uriSchemes()
     {
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
-        auto ret = std::vector<std::string>({"http", "https"});
-        if (forceHttp) ret.push_back("file");
+        auto ret = std::set<std::string>({"http", "https"});
+        if (forceHttp) ret.insert("file");
         return ret;
     }
 protected:
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index bbab452ab..6dc83e288 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -41,9 +41,9 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
 
     SSHMaster master;
 
-    static std::vector<std::string> uriPrefixes() { return {"ssh"}; }
+    static std::set<std::string> uriSchemes() { return {"ssh"}; }
 
-    LegacySSHStore(const string & host, const Params & params)
+    LegacySSHStore(const string & scheme, const string & host, const Params & params)
         : StoreConfig(params)
         , LegacySSHStoreConfig(params)
         , Store(params)
@@ -92,7 +92,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
 
     string getUri() override
     {
-        return uriPrefixes()[0] + "://" + host;
+        return *uriSchemes().begin() + "://" + host;
     }
 
     void queryPathInfoUncached(const StorePath & path,
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index becd74c75..4d11e0f2a 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -18,6 +18,7 @@ private:
 public:
 
     LocalBinaryCacheStore(
+        [[maybe_unused]] const std::string scheme,
         const Path & binaryCacheDir,
         const Params & params)
         : StoreConfig(params)
@@ -35,7 +36,7 @@ public:
         return "file://" + binaryCacheDir;
     }
 
-    static std::vector<std::string> uriPrefixes();
+    static std::set<std::string> uriSchemes();
 
 protected:
 
@@ -96,7 +97,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-std::vector<std::string> LocalBinaryCacheStore::uriPrefixes()
+std::set<std::string> LocalBinaryCacheStore::uriSchemes()
 {
     if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
         return {};
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 9bbb69b99..561502cbe 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -135,7 +135,10 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
 }
 
 
-UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
+UDSRemoteStore::UDSRemoteStore(
+        [[maybe_unused]] const std::string scheme,
+        std::string socket_path,
+        const Params & params)
     : UDSRemoteStore(params)
 {
     path.emplace(socket_path);
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 32daab59a..cceb8d185 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -165,11 +165,11 @@ class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual U
 public:
 
     UDSRemoteStore(const Params & params);
-    UDSRemoteStore(std::string path, const Params & params);
+    UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
 
     std::string getUri() override;
 
-    static std::vector<std::string> uriPrefixes()
+    static std::set<std::string> uriSchemes()
     { return {"unix"}; }
 
     bool sameMachine() override
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 30ef4082a..020b20c37 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -197,6 +197,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
     S3Helper s3Helper;
 
     S3BinaryCacheStoreImpl(
+        [[maybe_unused]] const std::string & scheme,
         const std::string & bucketName,
         const Params & params)
         : StoreConfig(params)
@@ -434,7 +435,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
         return paths;
     }
 
-    static std::vector<std::string> uriPrefixes() { return {"s3"}; }
+    static std::set<std::string> uriSchemes() { return {"s3"}; }
 
 };
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index da201a9c3..13265e127 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -22,7 +22,7 @@ class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
 {
 public:
 
-    SSHStore(const std::string & host, const Params & params)
+    SSHStore([[maybe_unused]] const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
         , Store(params)
         , RemoteStoreConfig(params)
@@ -38,11 +38,11 @@ public:
     {
     }
 
-    static std::vector<std::string> uriPrefixes() { return {"ssh-ng"}; }
+    static std::set<std::string> uriSchemes() { return {"ssh-ng"}; }
 
     std::string getUri() override
     {
-        return uriPrefixes()[0] + "://" + host;
+        return *uriSchemes().begin() + "://" + host;
     }
 
     bool sameMachine() override
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index fe3f5221c..9252c85e8 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1078,25 +1078,35 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
 ref<Store> openStore(const std::string & uri_,
     const Store::Params & extraParams)
 {
-    auto [uri, uriParams] = splitUriAndParams(uri_);
     auto params = extraParams;
-    params.insert(uriParams.begin(), uriParams.end());
+    try {
+        auto parsedUri = parseURL(uri_);
+        params.insert(parsedUri.query.begin(), parsedUri.query.end());
 
-    if (auto store = openFromNonUri(uri, params)) {
-        store->warnUnknownSettings();
-        return ref<Store>(store);
+        auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
+
+        for (auto implem : *Implementations::registered) {
+            if (implem.uriSchemes.count(parsedUri.scheme)) {
+                auto store = implem.create(parsedUri.scheme, baseURI, params);
+                if (store) {
+                    store->init();
+                    store->warnUnknownSettings();
+                    return ref<Store>(store);
+                }
+            }
+        }
     }
+    catch (BadURL) {
+        auto [uri, uriParams] = splitUriAndParams(uri_);
+        params.insert(uriParams.begin(), uriParams.end());
 
-    for (auto implem : *Implementations::registered) {
-        auto store = implem.create(uri, params);
-        if (store) {
-            store->init();
+        if (auto store = openFromNonUri(uri, params)) {
             store->warnUnknownSettings();
             return ref<Store>(store);
         }
     }
 
-    throw Error("don't know how to open Nix store '%s'", uri);
+    throw Error("don't know how to open Nix store '%s'", uri_);
 }
 
 std::list<ref<Store>> getDefaultSubstituters()
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8e9cb786c..bb7d8036b 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -760,7 +760,8 @@ std::list<ref<Store>> getDefaultSubstituters();
 
 struct StoreFactory
 {
-    std::function<std::shared_ptr<Store> (const std::string & uri, const Store::Params & params)> create;
+    std::set<std::string> uriSchemes;
+    std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
     std::function<std::shared_ptr<StoreConfig> ()> getConfig;
 };
 struct Implementations
@@ -773,13 +774,14 @@ struct Implementations
         if (!registered) registered = new std::vector<StoreFactory>();
         StoreFactory factory{
             .create =
-                ([](const std::string & uri, const Store::Params & params)
+                ([](const std::string & scheme, const std::string & uri, const Store::Params & params)
                  -> std::shared_ptr<Store>
-                 { return std::make_shared<T>(uri, params); }),
+                 { return std::make_shared<T>(scheme, uri, params); }),
             .getConfig =
                 ([]()
                  -> std::shared_ptr<StoreConfig>
-                 { return std::make_shared<TConfig>(); })
+                 { return std::make_shared<TConfig>(StringMap({})); }),
+            .uriSchemes = T::uriSchemes()
         };
         registered->push_back(factory);
     }

From 1129913c4e175f4890a7029d22a301676f4cc3ca Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Sep 2020 11:12:25 +0200
Subject: [PATCH 204/882] fixup! Correctly call all the parent contructors of
 the stores

---
 src/libstore/store-api.hh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bb7d8036b..6f7893e41 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -148,6 +148,8 @@ struct BuildResult
 struct StoreConfig : public Config
 {
     using Config::Config;
+    StoreConfig() = delete;
+
 
     const PathSetting storeDir_{this, false, settings.nixStore,
         "store", "path to the Nix store"};

From 29a632386e358b8cd72cf0545917e324bcfba16e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Sep 2020 11:13:41 +0200
Subject: [PATCH 205/882] fixup! Make the store plugins more introspectable

---
 src/libstore/store.hh | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 src/libstore/store.hh

diff --git a/src/libstore/store.hh b/src/libstore/store.hh
deleted file mode 100644
index bc73963f3..000000000
--- a/src/libstore/store.hh
+++ /dev/null
@@ -1,9 +0,0 @@
-#pragma once
-
-namespace nix {
-
-template<typename T> class BasicStore;
-class StoreConfig;
-typedef BasicStore<StoreConfig> Store;
-
-}

From d65962db4d5c1a2c8f456e3ae6bb789eb3a1e744 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Sep 2020 14:07:02 +0200
Subject: [PATCH 206/882] Make uri schemes grammar more RFC-compliant

Allow `-` and `.` in the RFC schemes as stated by
[RFC3986](https://tools.ietf.org/html/rfc3986#section-3.1).

Practically, this is needed so that `ssh-ng` is a valid URI scheme
---
 src/libutil/url.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 2ef88ef2a..1f716ba10 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -31,7 +31,7 @@ ParsedURL parseURL(const std::string & url);
 
 // URI stuff.
 const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
-const static std::string schemeRegex = "(?:[a-z+]+)";
+const static std::string schemeRegex = "(?:[a-z+.-]+)";
 const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
 const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
 const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";

From f24f0888f93b0881e2cefa1ceb4c8fc187c94f21 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Sep 2020 11:18:45 +0200
Subject: [PATCH 207/882] Document the new store hierarchy

---
 src/libstore/build.cc                    |  2 +-
 src/libstore/dummy-store.cc              |  3 +-
 src/libstore/http-binary-cache-store.cc  |  2 --
 src/libstore/legacy-ssh-store.cc         |  1 -
 src/libstore/local-binary-cache-store.cc |  2 --
 src/libstore/local-store.cc              |  2 --
 src/libstore/remote-store.cc             |  3 --
 src/libstore/s3-binary-cache-store.cc    |  2 --
 src/libstore/ssh-store.cc                |  2 --
 src/libstore/store-api.hh                | 46 +++++++++++++++++++++++-
 10 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4040a3fac..18bb9b12e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2883,7 +2883,7 @@ struct RestrictedStore : public LocalFSStore
     DerivationGoal & goal;
 
     RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), LocalFSStoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
+        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
     { }
 
     Path getRealStoreDir() override
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 210e5b11c..074953b1d 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -2,7 +2,7 @@
 
 namespace nix {
 
-struct DummyStoreConfig : StoreConfig {
+struct DummyStoreConfig : virtual StoreConfig {
     using StoreConfig::StoreConfig;
 };
 
@@ -14,7 +14,6 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
 
     DummyStore(const Params & params)
         : StoreConfig(params)
-        , DummyStoreConfig(params)
         , Store(params)
     {
     }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 095eaa82c..bd65273eb 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -33,9 +33,7 @@ public:
         const Path & _cacheUri,
         const Params & params)
         : StoreConfig(params)
-        , BinaryCacheStoreConfig(params)
         , BinaryCacheStore(params)
-        , HttpBinaryCacheStoreConfig(params)
         , cacheUri(scheme + "://" + _cacheUri)
     {
         if (cacheUri.back() == '/')
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 6dc83e288..ce9051d20 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -45,7 +45,6 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
 
     LegacySSHStore(const string & scheme, const string & host, const Params & params)
         : StoreConfig(params)
-        , LegacySSHStoreConfig(params)
         , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 4d11e0f2a..b893befbc 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -22,8 +22,6 @@ public:
         const Path & binaryCacheDir,
         const Params & params)
         : StoreConfig(params)
-        , BinaryCacheStoreConfig(params)
-        , LocalBinaryCacheStoreConfig(params)
         , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4e9840f3b..c618203f0 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -44,8 +44,6 @@ namespace nix {
 LocalStore::LocalStore(const Params & params)
     : StoreConfig(params)
     , Store(params)
-    , LocalFSStoreConfig(params)
-    , LocalStoreConfig(params)
     , LocalFSStore(params)
     , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
         "physical path to the Nix store"}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 561502cbe..4c8ffe71e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -126,9 +126,6 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : StoreConfig(params)
     , Store(params)
-    , LocalFSStoreConfig(params)
-    , RemoteStoreConfig(params)
-    , UDSRemoteStoreConfig(params)
     , LocalFSStore(params)
     , RemoteStore(params)
 {
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 020b20c37..e4548dd22 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -201,8 +201,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
         const std::string & bucketName,
         const Params & params)
         : StoreConfig(params)
-        , BinaryCacheStoreConfig(params)
-        , S3BinaryCacheStoreConfig(params)
         , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 13265e127..fac251d8f 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -25,9 +25,7 @@ public:
     SSHStore([[maybe_unused]] const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
         , Store(params)
-        , RemoteStoreConfig(params)
         , RemoteStore(params)
-        , SSHStoreConfig(params)
         , host(host)
         , master(
             host,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6f7893e41..a67123ff0 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -24,6 +24,31 @@
 
 namespace nix {
 
+/**
+ * About the class hierarchy of the store implementations:
+ *
+ * Each store type `Foo` consists of two classes:
+ *
+ * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
+ *   for the store
+ *
+ *   It should only contain members of type `const Setting<T>` (or subclasses
+ *   of it) and inherit the constructors of `StoreConfig`
+ *   (`using StoreConfig::StoreConfig`).
+ *
+ * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
+ *   implementation of the store.
+ *
+ *   This class is expected to have a constructor `Foo(const Params & params)`
+ *   that calls `StoreConfig(params)` (otherwise you're gonna encounter an
+ *   `assertion failure` when trying to instantiate it).
+ *
+ * You can then register the new store using:
+ *
+ * ```
+ * cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
+ * ```
+ */
 
 MakeError(SubstError, Error);
 MakeError(BuildError, Error); // denotes a permanent build failure
@@ -148,7 +173,26 @@ struct BuildResult
 struct StoreConfig : public Config
 {
     using Config::Config;
-    StoreConfig() = delete;
+
+    /**
+     * When constructing a store implementation, we pass in a map `params` of
+     * parameters that's supposed to initialize the associated config.
+     * To do that, we must use the `StoreConfig(StringMap & params)`
+     * constructor, so we'd like to `delete` its default constructor to enforce
+     * it.
+     *
+     * However, actually deleting it means that all the subclasses of
+     * `StoreConfig` will have their default constructor deleted (because it's
+     * supposed to call the deleted default constructor of `StoreConfig`). But
+     * because we're always using virtual inheritance, the constructors of
+     * child classes will never implicitely call this one, so deleting it will
+     * be more painful than anything else.
+     *
+     * So we `assert(false)` here to ensure at runtime that the right
+     * constructor is always called without having to redefine a custom
+     * constructor for each `*Config` class.
+     */
+    StoreConfig() { assert(false); }
 
 
     const PathSetting storeDir_{this, false, settings.nixStore,

From b73adacc1ebda8a790cd8e0b0988c3b0607d947a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Sep 2020 14:04:02 +0200
Subject: [PATCH 208/882] Add a name to the stores

So that it can be printed by `nix describe-stores`
---
 src/libstore/build.cc                    |  7 ++++++-
 src/libstore/dummy-store.cc              |  2 ++
 src/libstore/http-binary-cache-store.cc  |  2 ++
 src/libstore/legacy-ssh-store.cc         |  2 ++
 src/libstore/local-binary-cache-store.cc |  2 ++
 src/libstore/local-store.hh              |  2 ++
 src/libstore/remote-store.hh             |  2 ++
 src/libstore/s3-binary-cache-store.cc    |  2 ++
 src/libstore/ssh-store.cc                |  2 ++
 src/libstore/store-api.hh                |  1 +
 src/nix/describe-stores.cc               | 13 +++++++------
 11 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 18bb9b12e..6e55f83d5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2872,11 +2872,16 @@ void DerivationGoal::writeStructuredAttrs()
     chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
+struct RestrictedStoreConfig : LocalFSStoreConfig
+{
+    using LocalFSStoreConfig::LocalFSStoreConfig;
+    const std::string name() { return "Restricted Store"; }
+};
 
 /* A wrapper around LocalStore that only allows building/querying of
    paths that are in the input closures of the build or were added via
    recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore
+struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
 {
     ref<LocalStore> next;
 
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 074953b1d..128832e60 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -4,6 +4,8 @@ namespace nix {
 
 struct DummyStoreConfig : virtual StoreConfig {
     using StoreConfig::StoreConfig;
+
+    const std::string name() override { return "Dummy Store"; }
 };
 
 struct DummyStore : public Store, public virtual DummyStoreConfig
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index bd65273eb..f4ab15a10 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -10,6 +10,8 @@ MakeError(UploadToHTTP, Error);
 struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+
+    const std::string name() override { return "Http Binary Cache Store"; }
 };
 
 class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index ce9051d20..6563c001f 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -17,6 +17,8 @@ struct LegacySSHStoreConfig : virtual StoreConfig
     const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
     const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
+
+    const std::string name() override { return "Legacy SSH Store"; }
 };
 
 struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index b893befbc..357bc74df 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -7,6 +7,8 @@ namespace nix {
 struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+
+    const std::string name() override { return "Local Binary Cache Store"; }
 };
 
 class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 0265f0837..e7c9d1605 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -37,6 +37,8 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
     Setting<bool> requireSigs{(StoreConfig*) this,
         settings.requireSigs,
         "require-sigs", "whether store paths should have a trusted signature on import"};
+
+    const std::string name() override { return "Local Store"; }
 };
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index cceb8d185..a23690830 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -158,6 +158,8 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
         : UDSRemoteStoreConfig(Store::Params({}))
     {
     }
+
+    const std::string name() override { return "Local Daemon Store"; }
 };
 
 class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index e4548dd22..0b9c9bb31 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -186,6 +186,8 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
         this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
         this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
+
+    const std::string name() override { return "S3 Binary Cache Store"; }
 };
 
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index fac251d8f..df383bba6 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -16,6 +16,8 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
     const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
     const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
+
+    const std::string name() override { return "SSH Store"; }
 };
 
 class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a67123ff0..81b979b73 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -194,6 +194,7 @@ struct StoreConfig : public Config
      */
     StoreConfig() { assert(false); }
 
+    virtual const std::string name() = 0;
 
     const PathSetting storeDir_{this, false, settings.nixStore,
         "store", "path to the Nix store"};
diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc
index f019ec9c2..027187bd9 100644
--- a/src/nix/describe-stores.cc
+++ b/src/nix/describe-stores.cc
@@ -18,13 +18,14 @@ struct CmdDescribeStores : Command, MixJSON
 
     void run() override
     {
-
+        auto res = nlohmann::json::object();
+        for (auto & implem : *Implementations::registered) {
+            auto storeConfig = implem.getConfig();
+            auto storeName = storeConfig->name();
+            res[storeName] = storeConfig->toJSON();
+        }
         if (json) {
-            auto res = nlohmann::json::array();
-            for (auto & implem : *Implementations::registered) {
-                auto storeConfig = implem.getConfig();
-                std::cout << storeConfig->toJSON().dump() << std::endl;
-            }
+            std::cout << res;
         } else {
             throw Error("Only json is available for describe-stores");
         }

From 634cb2a5aec64164dd7fa9467d9cdb1569611c21 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Sep 2020 14:04:21 +0200
Subject: [PATCH 209/882] Add a markdown output to `nix describe-stores`

---
 src/nix/describe-stores.cc | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc
index 027187bd9..0cc2d9337 100644
--- a/src/nix/describe-stores.cc
+++ b/src/nix/describe-stores.cc
@@ -27,7 +27,16 @@ struct CmdDescribeStores : Command, MixJSON
         if (json) {
             std::cout << res;
         } else {
-            throw Error("Only json is available for describe-stores");
+            for (auto & [storeName, storeConfig] : res.items()) {
+                std::cout << "## " << storeName << std::endl << std::endl;
+                for (auto & [optionName, optionDesc] : storeConfig.items()) {
+                    std::cout << "### " << optionName << std::endl << std::endl;
+                    std::cout << optionDesc["description"].get<std::string>() << std::endl;
+                    std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
+                    if (!optionDesc["aliases"].empty())
+                        std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
+                }
+            }
         }
     }
 };

From 888f7afe9fe1a69c27bcc15a3e0fb0be26a11172 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Sep 2020 14:12:47 +0200
Subject: [PATCH 210/882] Fix build issues with gcc

---
 src/libstore/local-binary-cache-store.cc | 2 +-
 src/libstore/remote-store.cc             | 2 +-
 src/libstore/s3-binary-cache-store.cc    | 2 +-
 src/libstore/ssh-store.cc                | 2 +-
 src/libstore/store-api.hh                | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 357bc74df..b5744448e 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -20,7 +20,7 @@ private:
 public:
 
     LocalBinaryCacheStore(
-        [[maybe_unused]] const std::string scheme,
+        const std::string scheme,
         const Path & binaryCacheDir,
         const Params & params)
         : StoreConfig(params)
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 4c8ffe71e..1abe236f7 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -133,7 +133,7 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
 
 
 UDSRemoteStore::UDSRemoteStore(
-        [[maybe_unused]] const std::string scheme,
+        const std::string scheme,
         std::string socket_path,
         const Params & params)
     : UDSRemoteStore(params)
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 0b9c9bb31..b3541d7b8 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -199,7 +199,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
     S3Helper s3Helper;
 
     S3BinaryCacheStoreImpl(
-        [[maybe_unused]] const std::string & scheme,
+        const std::string & scheme,
         const std::string & bucketName,
         const Params & params)
         : StoreConfig(params)
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index df383bba6..8b6e48fb0 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -24,7 +24,7 @@ class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
 {
 public:
 
-    SSHStore([[maybe_unused]] const std::string & scheme, const std::string & host, const Params & params)
+    SSHStore(const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
         , Store(params)
         , RemoteStore(params)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 81b979b73..1bea4837b 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -820,6 +820,7 @@ struct Implementations
     {
         if (!registered) registered = new std::vector<StoreFactory>();
         StoreFactory factory{
+            .uriSchemes = T::uriSchemes(),
             .create =
                 ([](const std::string & scheme, const std::string & uri, const Store::Params & params)
                  -> std::shared_ptr<Store>
@@ -827,8 +828,7 @@ struct Implementations
             .getConfig =
                 ([]()
                  -> std::shared_ptr<StoreConfig>
-                 { return std::make_shared<TConfig>(StringMap({})); }),
-            .uriSchemes = T::uriSchemes()
+                 { return std::make_shared<TConfig>(StringMap({})); })
         };
         registered->push_back(factory);
     }

From a1e82ba450f274966dc45375b42b5d4413fff77d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Sep 2020 15:35:17 +0200
Subject: [PATCH 211/882] fixup! Add a default value for the settings

---
 src/libutil/tests/config.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index c5abefe11..c7777a21f 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -161,7 +161,7 @@ namespace nix {
         Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
         setting.assign("value");
 
-        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#");
+        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#");
     }
 
     TEST(Config, setSettingAlias) {

From fc2d31c423377cea2355e7f7a4ce75e64a7f3620 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 15 Sep 2020 09:10:37 +0200
Subject: [PATCH 212/882] Add `(StoreConfig*)` casts to work around a GCC bug

Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 that was
already there in the code but was accidentally removed in the last
commits
---
 src/libstore/binary-cache-store.hh | 12 ++++++------
 src/libstore/legacy-ssh-store.cc   | 16 ++++++++--------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index f1fc82013..4b779cdd4 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -15,12 +15,12 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
-    const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
-    const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
-    const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
-    const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
-    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
+    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
+    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
+    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
         "enable multi-threading compression, available for xz only currently"};
 };
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 6563c001f..e9478c1d5 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -12,11 +12,11 @@ namespace nix {
 struct LegacySSHStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
-    const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
-    const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
-    const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
-    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
+    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
+    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
+    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
 
     const std::string name() override { return "Legacy SSH Store"; }
 };
@@ -24,9 +24,9 @@ struct LegacySSHStoreConfig : virtual StoreConfig
 struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
 {
     // Hack for getting remote build log output.
-    // Intentionally not in `StoreConfig` so that it doesn't appear in the
-    // documentation
-    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
+    // the documentation
+    const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
 
     struct Connection
     {

From 93c0e14a308d513cf61daa7a003417e42a7dc2f8 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 15 Sep 2020 11:34:20 +0200
Subject: [PATCH 213/882] Include the full nlohmann/json header in config.hh

It is apparently required for using `toJSONObject()`, which we do inside
the header file (because it's in a template).

This was accidentally working when building Nix itself (presumably because
`config.hh` was always included after `nlohman/json.hpp`) but caused a
(pretty dirty) build failure in the perl bindings package.
---
 src/libutil/config.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 42fc856e0..18fa9dea8 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -4,7 +4,7 @@
 
 #include "types.hh"
 
-#include <nlohmann/json_fwd.hpp>
+#include <nlohmann/json.hpp>
 
 #pragma once
 

From e0817cbcdcdbec81d7ce2f5141b4f99bbc2bece7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 16 Sep 2020 09:06:35 +0200
Subject: [PATCH 214/882] Don't include nlohmann/json.hpp in config.hh

Instead make a separate header with the template implementation of
`BaseSetting<T>::toJSONObj` that can be included where needed
---
 src/libstore/globals.hh              |  1 +
 src/libutil/abstractsettingtojson.hh | 15 +++++++++++++++
 src/libutil/config.cc                |  1 +
 src/libutil/config.hh                | 10 ++--------
 4 files changed, 19 insertions(+), 8 deletions(-)
 create mode 100644 src/libutil/abstractsettingtojson.hh

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 8a2d3ff75..02721285a 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "config.hh"
+#include "abstractsettingtojson.hh"
 #include "util.hh"
 
 #include <map>
diff --git a/src/libutil/abstractsettingtojson.hh b/src/libutil/abstractsettingtojson.hh
new file mode 100644
index 000000000..b3fbc84f7
--- /dev/null
+++ b/src/libutil/abstractsettingtojson.hh
@@ -0,0 +1,15 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+#include "config.hh"
+
+namespace nix {
+template<typename T>
+std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
+{
+    auto obj = AbstractSetting::toJSONObject();
+    obj.emplace("value", value);
+    obj.emplace("defaultValue", defaultValue);
+    return obj;
+}
+}
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index faa5cdbeb..309d23b40 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -1,5 +1,6 @@
 #include "config.hh"
 #include "args.hh"
+#include "abstractsettingtojson.hh"
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 18fa9dea8..1f5f4e7b9 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -4,7 +4,7 @@
 
 #include "types.hh"
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 #pragma once
 
@@ -255,13 +255,7 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    std::map<std::string, nlohmann::json> toJSONObject() override
-    {
-        auto obj = AbstractSetting::toJSONObject();
-        obj.emplace("value", value);
-        obj.emplace("defaultValue", defaultValue);
-        return obj;
-    }
+    std::map<std::string, nlohmann::json> toJSONObject() override;
 };
 
 template<typename T>

From d72927aa7a16be56b134dfadf34d5bbdb751f264 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 16 Sep 2020 13:51:53 +0200
Subject: [PATCH 215/882] Fix the s3 store

Add some necessary casts in the initialisation of the store's config
---
 src/libstore/s3-binary-cache-store.cc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index b3541d7b8..d43f267e0 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -175,17 +175,17 @@ S3Helper::FileTransferResult S3Helper::getObject(
 struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
-    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
-    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
-    const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
-    const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
-    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
-    const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
-    const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
+    const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
+    const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
+    const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
+    const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
+    const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
     const Setting<bool> multipartUpload{
-        this, false, "multipart-upload", "whether to use multi-part uploads"};
+        (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
-        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
+        (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
     const std::string name() override { return "S3 Binary Cache Store"; }
 };

From c29624bf7d58aeb698ddd12b983465d355fccf79 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 16 Sep 2020 13:52:15 +0200
Subject: [PATCH 216/882] Add a test for `nix describe-stores`

Doesn't test much, but at least ensures that the command runs properly
---
 tests/describe-stores.sh | 8 ++++++++
 tests/local.mk           | 1 +
 2 files changed, 9 insertions(+)
 create mode 100644 tests/describe-stores.sh

diff --git a/tests/describe-stores.sh b/tests/describe-stores.sh
new file mode 100644
index 000000000..3fea61483
--- /dev/null
+++ b/tests/describe-stores.sh
@@ -0,0 +1,8 @@
+source common.sh
+
+# Query an arbitrary value in `nix describe-stores --json`'s output just to
+# check that it has the right structure
+[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]]
+
+# Ensure that the output of `nix describe-stores` isn't empty
+[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]]
diff --git a/tests/local.mk b/tests/local.mk
index 5d25de019..594901504 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -32,6 +32,7 @@ nix_tests = \
   post-hook.sh \
   function-trace.sh \
   recursive.sh \
+  describe-stores.sh \
   flakes.sh \
   content-addressed.sh
   # parallel.sh

From 77a0e2c5beba478f4cfc9f3c9516e516a5da43c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Wed, 16 Sep 2020 14:00:21 +0200
Subject: [PATCH 217/882] Remove useless exception copy

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9252c85e8..76cbc0605 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1096,7 +1096,7 @@ ref<Store> openStore(const std::string & uri_,
             }
         }
     }
-    catch (BadURL) {
+    catch (BadURL &) {
         auto [uri, uriParams] = splitUriAndParams(uri_);
         params.insert(uriParams.begin(), uriParams.end());
 

From 2eacc1bc00fcc3bccd470b846062c63bc408be05 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Sep 2020 14:18:46 +0200
Subject: [PATCH 218/882] builtins.toFile: Fix indentation

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4a0dd5544..816980f10 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1704,7 +1704,7 @@ static RegisterPrimOp primop_toFile({
         ...
         cp ${configFile} $out/etc/foo.conf
       ";
-    ```
+      ```
 
       Note that `${configFile}` is an
       [antiquotation](language-values.md), so the result of the

From 39bc49318f64d30b945d9dbb4f96f225d25dbd50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Sep 2020 14:55:24 +0200
Subject: [PATCH 219/882] jq -> nix

---
 doc/manual/generate-builtins.jq  |  6 ----
 doc/manual/generate-builtins.nix | 14 ++++++++
 doc/manual/generate-manpage.jq   | 44 -------------------------
 doc/manual/generate-manpage.nix  | 56 ++++++++++++++++++++++++++++++++
 doc/manual/generate-options.jq   | 16 ---------
 doc/manual/generate-options.nix  | 21 ++++++++++++
 doc/manual/local.mk              | 16 ++++-----
 doc/manual/utils.nix             |  7 ++++
 8 files changed, 106 insertions(+), 74 deletions(-)
 delete mode 100644 doc/manual/generate-builtins.jq
 create mode 100644 doc/manual/generate-builtins.nix
 delete mode 100644 doc/manual/generate-manpage.jq
 create mode 100644 doc/manual/generate-manpage.nix
 delete mode 100644 doc/manual/generate-options.jq
 create mode 100644 doc/manual/generate-options.nix
 create mode 100644 doc/manual/utils.nix

diff --git a/doc/manual/generate-builtins.jq b/doc/manual/generate-builtins.jq
deleted file mode 100644
index c38799479..000000000
--- a/doc/manual/generate-builtins.jq
+++ /dev/null
@@ -1,6 +0,0 @@
-. | to_entries | sort_by(.key) | map(
-  "  - `builtins." + .key + "` "
-  + (.value.args | map("*" + . + "*") | join(" "))
-  + "  \n\n"
-  + (.value.doc | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
-) | join("")
diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
new file mode 100644
index 000000000..416a7fdba
--- /dev/null
+++ b/doc/manual/generate-builtins.nix
@@ -0,0 +1,14 @@
+with builtins;
+with import ./utils.nix;
+
+builtins:
+
+concatStrings (map
+  (name:
+    let builtin = builtins.${name}; in
+    "  - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args)
+    + "  \n\n"
+    + concatStrings (map (s: "    ${s}\n") (splitLines builtin.doc)) + "\n\n"
+  )
+  (attrNames builtins))
+
diff --git a/doc/manual/generate-manpage.jq b/doc/manual/generate-manpage.jq
deleted file mode 100644
index dd632f162..000000000
--- a/doc/manual/generate-manpage.jq
+++ /dev/null
@@ -1,44 +0,0 @@
-def show_flags:
-  .flags
-  | map_values(select(.category != "config"))
-  | to_entries
-  | map(
-      "  - `--" + .key + "`"
-      + (if .value.shortName then " / `" + .value.shortName + "`" else "" end)
-      + (if .value.labels then " " + (.value.labels | map("*" + . + "*") | join(" ")) else "" end)
-      + "  \n"
-      + "    " + .value.description + "\n\n")
-  | join("")
-  ;
-
-def show_synopsis:
-  "`" + .command + "` [*flags*...] " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n\n"
-  ;
-
-def show_command:
-  . as $top |
-  .section + " Name\n\n"
-  + "`" + .command + "` - " + .def.description + "\n\n"
-  + .section + " Synopsis\n\n"
-  + ({"command": .command, "args": .def.args} | show_synopsis)
-  + (if .def | has("doc")
-     then .section + " Description\n\n" + .def.doc + "\n\n"
-     else ""
-     end)
-  + (if (.def.flags | length) > 0 then
-      .section + " Flags\n\n"
-      + (.def | show_flags)
-    else "" end)
-  + (if (.def.examples | length) > 0 then
-      .section + " Examples\n\n"
-      + (.def.examples | map(.description + "\n\n```console\n" + .command + "\n```\n" ) | join("\n"))
-      + "\n"
-     else "" end)
-  + (if .def.commands then .def.commands | to_entries | map(
-      "# Subcommand `" + ($top.command + " " + .key) + "`\n\n"
-      + ({"command": ($top.command + " " + .key), "section": "##", "def": .value} | show_command)
-    ) | join("") else "" end)
-  ;
-
-"Title: nix\n\n"
-+ ({"command": "nix", "section": "#", "def": .} | show_command)
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
new file mode 100644
index 000000000..4709c0e2c
--- /dev/null
+++ b/doc/manual/generate-manpage.nix
@@ -0,0 +1,56 @@
+with builtins;
+with import ./utils.nix;
+
+let
+
+  showCommand =
+    { command, section, def }:
+    "${section} Name\n\n"
+    + "`${command}` - ${def.description}\n\n"
+    + "${section} Synopsis\n\n"
+    + showSynopsis { inherit command; args = def.args; }
+    + (if def ? doc
+       then "${section} Description\n\n" + def.doc + "\n\n"
+       else "")
+    + (let s = showFlags def.flags; in
+       if s != ""
+       then "${section} Flags\n\n${s}"
+       else "")
+    + (if def.examples or [] != []
+       then
+         "${section} Examples\n\n"
+         + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
+       else "")
+    + (if def.commands or [] != []
+       then concatStrings (
+         map (name:
+           "# Subcommand `${command} ${name}`\n\n"
+           + showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; })
+           (attrNames def.commands))
+       else "");
+
+  showFlags = flags:
+    concatStrings
+      (map (longName:
+        let flag = flags.${longName}; in
+        if flag.category or "" != "config"
+        then
+          "  - `--${longName}`"
+          + (if flag ? shortName then " / `${flag.shortName}`" else "")
+          + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
+          + "  \n"
+          + "    " + flag.description + "\n\n"
+        else "")
+        (attrNames flags));
+
+  showSynopsis =
+    { command, args }:
+    "`${command}` [*flags*...] ${concatStringsSep " "
+      (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
+
+in
+
+command:
+
+"Title: nix\n\n"
++ showCommand { command = "nix"; section = "#"; def = command; }
diff --git a/doc/manual/generate-options.jq b/doc/manual/generate-options.jq
deleted file mode 100644
index ccf62e8ed..000000000
--- a/doc/manual/generate-options.jq
+++ /dev/null
@@ -1,16 +0,0 @@
-. | to_entries | sort_by(.key) | map(
-  "  - `" + .key + "`  \n\n"
-  + (.value.description | split("\n") | map("    " + . + "\n") | join("")) + "\n\n"
-  + "    **Default:** " + (
-      if .value.value == "" or .value.value == []
-      then "*empty*"
-      elif (.value.value | type) == "array"
-      then "`" + (.value.value | join(" ")) + "`"
-      else "`" + (.value.value | tostring) + "`"
-      end)
-  + "\n\n"
-  + (if (.value.aliases | length) > 0
-     then "    **Deprecated alias:** " + (.value.aliases | map("`" + . + "`") | join(", ")) + "\n\n"
-     else ""
-     end)
-) | join("")
diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix
new file mode 100644
index 000000000..7afe279c3
--- /dev/null
+++ b/doc/manual/generate-options.nix
@@ -0,0 +1,21 @@
+with builtins;
+with import ./utils.nix;
+
+options:
+
+concatStrings (map
+  (name:
+    let option = options.${name}; in
+    "  - `${name}`  \n\n"
+    + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
+    + "    **Default:** " + (
+      if option.value == "" || option.value == []
+      then "*empty*"
+      else if isBool option.value
+      then (if option.value then "`true`" else "`false`")
+      else "`" + toString option.value + "`") + "\n\n"
+    + (if option.aliases != []
+       then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
+       else "")
+    )
+  (attrNames options))
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 297a73414..c219153e9 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -24,12 +24,12 @@ $(d)/%.8: $(d)/src/command-ref/%.md
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
-$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.jq
-	jq -r -f doc/manual/generate-manpage.jq $< > $@
+$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
+	$(trace-gen) $(bindir)/nix eval --experimental-features nix-command --impure --raw --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.jq $(d)/src/command-ref/conf-file-prefix.md
-	cat doc/manual/src/command-ref/conf-file-prefix.md > $@
-	jq -r -f doc/manual/generate-options.jq $< >> $@
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
+	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@
+	$(trace-gen) $(bindir)/nix eval --experimental-features nix-command --impure --raw --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@
 
 $(d)/nix.json: $(bindir)/nix
 	$(trace-gen) $(bindir)/nix __dump-args > $@
@@ -37,9 +37,9 @@ $(d)/nix.json: $(bindir)/nix
 $(d)/conf-file.json: $(bindir)/nix
 	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@
 
-$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.jq $(d)/src/expressions/builtins-prefix.md
-	cat doc/manual/src/expressions/builtins-prefix.md > $@
-	jq -r -f doc/manual/generate-builtins.jq $< >> $@
+$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
+	@cat doc/manual/src/expressions/builtins-prefix.md > $@
+	$(trace-gen) $(bindir)/nix eval --experimental-features nix-command --impure --raw --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@
 
 $(d)/builtins.json: $(bindir)/nix
 	$(trace-gen) $(bindir)/nix __dump-builtins > $@
diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix
new file mode 100644
index 000000000..50150bf3e
--- /dev/null
+++ b/doc/manual/utils.nix
@@ -0,0 +1,7 @@
+with builtins;
+
+{
+  splitLines = s: filter (x: !isList x) (split "\n" s);
+
+  concatStrings = concatStringsSep "";
+}

From 0066ef6c59c356d5714f2e26af9471937ba0c865 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Sep 2020 16:56:28 +0200
Subject: [PATCH 220/882] Fix doc generation

---
 doc/manual/local.mk    | 24 ++++++++++++++++--------
 src/libexpr/eval.cc    |  6 +++++-
 src/libexpr/primops.cc | 11 +++++++----
 src/nix/main.cc        |  1 +
 4 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index c219153e9..3b8e7e2df 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -15,6 +15,8 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 dist-files += $(man-pages)
 
+nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
+
 $(d)/%.1: $(d)/src/command-ref/%.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
@@ -25,24 +27,30 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	$(trace-gen) lowdown -sT man $^ -o $@
 
 $(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
-	$(trace-gen) $(bindir)/nix eval --experimental-features nix-command --impure --raw --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@
+	$(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp
+	@mv $@.tmp $@
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
-	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@
-	$(trace-gen) $(bindir)/nix eval --experimental-features nix-command --impure --raw --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@
+	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
+	$(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(bindir)/nix __dump-args > $@
+	$(trace-gen) $(bindir)/nix __dump-args > $@.tmp
+	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@
+	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+	@mv $@.tmp $@
 
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
-	@cat doc/manual/src/expressions/builtins-prefix.md > $@
-	$(trace-gen) $(bindir)/nix eval --experimental-features nix-command --impure --raw --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@
+	@cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
+	$(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	@mv $@.tmp $@
 
 $(d)/builtins.json: $(bindir)/nix
-	$(trace-gen) $(bindir)/nix __dump-builtins > $@
+	$(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
+	mv $@.tmp $@
 
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d678231c6..139067f20 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -370,7 +370,11 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
-    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
+
+    try {
+        addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
+    } catch (Error &) {
+    }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 816980f10..d0b0c57b2 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3565,10 +3565,13 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
-    sDerivationNix = symbols.create(path);
-    evalFile(path, v);
-    addConstant("derivation", v);
+    try {
+        string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
+        sDerivationNix = symbols.create(path);
+        evalFile(path, v);
+        addConstant("derivation", v);
+    } catch (SysError &) {
+    }
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
diff --git a/src/nix/main.cc b/src/nix/main.cc
index e9479f564..1e9e07bc0 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -185,6 +185,7 @@ void mainWrapped(int argc, char * * argv)
     }
 
     if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+        evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));
         auto res = nlohmann::json::object();
         auto builtins = state.baseEnv.values[0]->attrs;

From b7c02232b2c2c51b3b81389b247e400b7a115abc Mon Sep 17 00:00:00 2001
From: Marwan Aljubeh <marwan.aljubeh@gmail.com>
Date: Wed, 16 Sep 2020 17:56:43 +0100
Subject: [PATCH 221/882] Fix the nix-daemon Mac OS SSL CA cert

Mac OS multi-user installations are currently broken because all requests
made by nix-daemon to the binary cache fail with:

```
unable to download ... Problem with the SSL CA cert (path? access rights?) (77).
```

This change ensures that the nix-daemon knows where to find the SSL CA cert file.

Fixes #2899 and #3261.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 9f26296a9..c334639e2 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -4,6 +4,8 @@
   <dict>
     <key>EnvironmentVariables</key>
     <dict>
+      <key>NIX_SSL_CERT_FILE</key>
+      <string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string>
       <key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key>
       <string>YES</string>
     </dict>

From a303c0b6dc71b1e0d6a57986c3f7a9b61361cd92 Mon Sep 17 00:00:00 2001
From: Greg Hale <imalsogreg@gmail.com>
Date: Wed, 17 Jun 2020 15:08:59 -0400
Subject: [PATCH 222/882] Fetch commits from github/gitlab  using Auth header

`nix flake info` calls the github 'commits' API, which requires
authorization when the repository is private. Currently this request
fails with a 404.

This commit adds an authorization header when calling the 'commits' API.
It also changes the way that the 'tarball' API authenticates, moving the
user's token from a query parameter into the Authorization header.

The query parameter method is recently deprecated and will be disallowed
in November 2020. Using them today triggers a warning email.
---
 src/libexpr/common-eval-args.cc  |  2 +-
 src/libexpr/parser.y             |  2 +-
 src/libexpr/primops/fetchTree.cc |  4 +-
 src/libfetchers/fetchers.hh      |  2 +
 src/libfetchers/github.cc        | 76 +++++++++++++++++++++++---------
 src/libfetchers/registry.cc      |  2 +-
 src/libfetchers/tarball.cc       |  9 ++--
 src/libstore/filetransfer.cc     |  3 ++
 src/libstore/filetransfer.hh     |  4 ++
 src/libstore/globals.hh          |  3 ++
 src/libutil/types.hh             |  2 +
 src/nix-channel/nix-channel.cc   |  6 +--
 12 files changed, 84 insertions(+), 31 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 10c1a6975..d71aa22f1 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
     if (isUri(s)) {
         return state.store->toRealPath(
             fetchers::downloadTarball(
-                state.store, resolveUri(s), "source", false).first.storePath);
+                state.store, resolveUri(s), Headers {}, "source", false).first.storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 24b21f7da..28e31f46b 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -719,7 +719,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
     if (isUri(elem.second)) {
         try {
             res = { true, store->toRealPath(fetchers::downloadTarball(
-                        store, resolveUri(elem.second), "source", false).first.storePath) };
+                        store, resolveUri(elem.second), Headers {}, "source", false).first.storePath) };
         } catch (FileTransferError & e) {
             logWarning({
                 .name = "Entry download",
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 06e8304b8..3001957b4 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -201,8 +201,8 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
-        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+        ? fetchers::downloadTarball(state.store, *url, Headers {}, name, (bool) expectedHash).first.storePath
+        : fetchers::downloadFile(state.store, *url, Headers{}, name, (bool) expectedHash).storePath;
 
     auto path = state.store->toRealPath(storePath);
 
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 89b1e6e7d..62807e53b 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -118,12 +118,14 @@ struct DownloadFileResult
 DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
+    const Headers & headers,
     const std::string & name,
     bool immutable);
 
 std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
+    const Headers & headers,
     const std::string & name,
     bool immutable);
 
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 1cc0c5e2e..d8d0351b9 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -3,11 +3,24 @@
 #include "fetchers.hh"
 #include "globals.hh"
 #include "store-api.hh"
+#include "types.hh"
 
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
 
+struct DownloadUrl
+{
+    std::string url;
+    std::optional<std::pair<std::string, std::string>> access_token_header;
+
+    DownloadUrl(const std::string & url)
+        : url(url) { }
+
+    DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
+        : url(url), access_token_header(access_token_header) { }
+};
+
 // A github or gitlab url
 const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
 std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
@@ -16,6 +29,8 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::string type() = 0;
 
+    virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
+
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
         if (url.scheme != type()) return {};
@@ -131,7 +146,7 @@ struct GitArchiveInputScheme : InputScheme
 
     virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
 
-    virtual std::string getDownloadUrl(const Input & input) const = 0;
+    virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
@@ -160,7 +175,12 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url, "source", true);
+        Headers headers;
+        if (url.access_token_header) {
+            headers.push_back(*url.access_token_header);
+        }
+
+        auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
 
         input.attrs.insert_or_assign("lastModified", lastModified);
 
@@ -182,11 +202,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    void addAccessToken(std::string & url) const
-    {
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;
+    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+        return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
@@ -195,18 +212,21 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        addAccessToken(url);
+        Headers headers;
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            headers.push_back(accessHeaderFromToken(accessToken));
 
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", false).storePath)));
+                    downloadFile(store, url, headers, "source", false).storePath)));
         auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
     }
 
-    std::string getDownloadUrl(const Input & input) const override
+    DownloadUrl getDownloadUrl(const Input & input) const override
     {
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
@@ -215,9 +235,13 @@ struct GitHubInputScheme : GitArchiveInputScheme
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        addAccessToken(url);
-
-        return url;
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "") {
+            auto auth_header = accessHeaderFromToken(accessToken);
+            return DownloadUrl(url, auth_header);
+        } else {
+            return DownloadUrl(url);
+        }
     }
 
     void clone(const Input & input, const Path & destDir) override
@@ -234,21 +258,31 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
+    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+    }
+
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
         auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+
+        Headers headers;
+        std::string accessToken = settings.gitlabAccessToken.get();
+        if (accessToken != "")
+            headers.push_back(accessHeaderFromToken(accessToken));
+
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", false).storePath)));
+                    downloadFile(store, url, headers, "source", false).storePath)));
         auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
     }
 
-    std::string getDownloadUrl(const Input & input) const override
+    DownloadUrl getDownloadUrl(const Input & input) const override
     {
         // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
         auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
@@ -256,12 +290,14 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;*/
+        std::string accessToken = settings.gitlabAccessToken.get();
+        if (accessToken != "") {
+            auto auth_header = accessHeaderFromToken(accessToken);
+            return DownloadUrl(url, auth_header);
+        } else {
+            return DownloadUrl(url);
+        }
 
-        return url;
     }
 
     void clone(const Input & input, const Path & destDir) override
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 4367ee810..551e7684a 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -145,7 +145,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
         auto path = settings.flakeRegistry.get();
 
         if (!hasPrefix(path, "/")) {
-            auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
+            auto storePath = downloadFile(store, path, Headers {}, "flake-registry.json", false).storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
             path = store->toRealPath(storePath);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index a2d16365e..cf6d6e3d2 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -5,12 +5,14 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "tarfile.hh"
+#include "types.hh"
 
 namespace nix::fetchers {
 
 DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
+    const Headers & headers,
     const std::string & name,
     bool immutable)
 {
@@ -36,7 +38,7 @@ DownloadFileResult downloadFile(
     if (cached && !cached->expired)
         return useCached();
 
-    FileTransferRequest request(url);
+    FileTransferRequest request(url, headers);
     if (cached)
         request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
     FileTransferResult res;
@@ -110,6 +112,7 @@ DownloadFileResult downloadFile(
 std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
+    const Headers & headers,
     const std::string & name,
     bool immutable)
 {
@@ -127,7 +130,7 @@ std::pair<Tree, time_t> downloadTarball(
             getIntAttr(cached->infoAttrs, "lastModified")
         };
 
-    auto res = downloadFile(store, url, name, immutable);
+    auto res = downloadFile(store, url, headers, name, immutable);
 
     std::optional<StorePath> unpackedStorePath;
     time_t lastModified;
@@ -222,7 +225,7 @@ struct TarballInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), Headers {}, "source", false).first;
         return {std::move(tree), input};
     }
 };
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 4149f8155..13ed429fa 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -112,6 +112,9 @@ struct curlFileTransfer : public FileTransfer
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
             if (!request.mimeType.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
+            for (auto it = request.headers.begin(); it != request.headers.end(); ++it){
+                requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
+            }
         }
 
         ~TransferItem()
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index 0d608c8d8..7e302ff39 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -51,6 +51,7 @@ extern FileTransferSettings fileTransferSettings;
 struct FileTransferRequest
 {
     std::string uri;
+    Headers headers;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
@@ -65,6 +66,9 @@ struct FileTransferRequest
     FileTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
+    FileTransferRequest(const std::string & uri, Headers headers)
+        : uri(uri), headers(headers) { }
+
     std::string verb()
     {
         return data ? "upload" : "download";
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 02721285a..b2e7610ee 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -863,6 +863,9 @@ public:
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
         "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
 
+    Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token",
+        "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."};
+
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 3af485fa0..55d02bcf9 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -24,6 +24,8 @@ typedef string Path;
 typedef list<Path> Paths;
 typedef set<Path> PathSet;
 
+typedef vector<std::pair<string, string>> Headers;
+
 /* Helper class to run code at startup. */
 template<typename T>
 struct OnStartup
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 3ccf620c9..94d33a75c 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -87,7 +87,7 @@ static void update(const StringSet & channelNames)
         // We want to download the url to a file to see if it's a tarball while also checking if we
         // got redirected in the process, so that we can grab the various parts of a nix channel
         // definition from a consistent location if the redirect changes mid-download.
-        auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
+        auto result = fetchers::downloadFile(store, url, Headers {}, std::string(baseNameOf(url)), false);
         auto filename = store->toRealPath(result.storePath);
         url = result.effectiveUrl;
 
@@ -112,9 +112,9 @@ static void update(const StringSet & channelNames)
         if (!unpacked) {
             // Download the channel tarball.
             try {
-                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
+                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", Headers {}, "nixexprs.tar.xz", false).storePath);
             } catch (FileTransferError & e) {
-                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
+                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", Headers {}, "nixexprs.tar.bz2", false).storePath);
             }
         }
 

From 7fdbb377ba800728a47095008cec11be7d970330 Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Wed, 9 Sep 2020 14:55:43 -0400
Subject: [PATCH 223/882] Start to fix floating CA + remote building

---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ee12f8e67..32980f264 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4613,7 +4613,7 @@ void DerivationGoal::flushLine()
 
 std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
 {
-    if (drv->type() != DerivationType::CAFloating) {
+    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
         std::map<std::string, std::optional<StorePath>> res;
         for (auto & [name, output] : drv->outputs)
             res.insert_or_assign(name, output.path(worker.store, drv->name, name));
@@ -4625,7 +4625,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
 
 OutputPathMap DerivationGoal::queryDerivationOutputMap()
 {
-    if (drv->type() != DerivationType::CAFloating) {
+    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
         OutputPathMap res;
         for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
             res.insert_or_assign(name, *output.second);

From 2741fffa350ec59d29ade24dd93007d535a61bde Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 9 Sep 2020 19:13:21 +0000
Subject: [PATCH 224/882] Ensure resolved CA derivations are written

so we can link outputs to deriver and thus properly cache.
---
 src/libstore/build.cc       | 33 ++++++++++++++++++++-------------
 src/libstore/derivations.hh |  1 +
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 32980f264..87c50f0e6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4298,11 +4298,13 @@ void DerivationGoal::registerOutputs()
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
-    ValidPathInfos infos2;
-    for (auto & [outputName, newInfo] : infos) {
-        infos2.push_back(newInfo);
+    {
+        ValidPathInfos infos2;
+        for (auto & [outputName, newInfo] : infos) {
+            infos2.push_back(newInfo);
+        }
+        worker.store.registerValidPaths(infos2);
     }
-    worker.store.registerValidPaths(infos2);
 
     /* In case of a fixed-output derivation hash mismatch, throw an
        exception now that we have registered the output as valid. */
@@ -4314,16 +4316,21 @@ void DerivationGoal::registerOutputs()
        means it's safe to link the derivation to the output hash. We must do
        that for floating CA derivations, which otherwise couldn't be cached,
        but it's fine to do in all cases. */
-    for (auto & [outputName, newInfo] : infos) {
-        if (useDerivation)
-            worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
-        else {
-            /* Once a floating CA derivations reaches this point, it must
-               already be resolved, drvPath the basic derivation path, and
-               a file existsing at that path for sake of the DB's foreign key. */
-            assert(drv->type() != DerivationType::CAFloating);
-        }
+    bool isCaFloating = drv->type() == DerivationType::CAFloating;
+
+    auto drvPath2 = drvPath;
+    if (!useDerivation && isCaFloating) {
+        /* Once a floating CA derivations reaches this point, it
+           must already be resolved, so we don't bother trying to
+           downcast drv to get would would just be an empty
+           inputDrvs field. */
+        Derivation drv2 { *drv };
+        drvPath2 = writeDerivation(worker.store, drv2);
     }
+
+    if (useDerivation || isCaFloating)
+        for (auto & [outputName, newInfo] : infos)
+            worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
 }
 
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index adbf8c094..74601134e 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -136,6 +136,7 @@ struct Derivation : BasicDerivation
     std::optional<BasicDerivation> tryResolve(Store & store);
 
     Derivation() = default;
+    Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 };
 

From 10d1865f5f443ddd57bb408a99f0afd74436e963 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Sep 2020 09:12:39 +0200
Subject: [PATCH 225/882] Remove corepkgs/derivation.nix

---
 corepkgs/local.mk                                |  1 -
 src/libexpr/local.mk                             |  2 +-
 src/libexpr/primops.cc                           | 12 +++++-------
 {corepkgs => src/libexpr/primops}/derivation.nix |  0
 4 files changed, 6 insertions(+), 9 deletions(-)
 rename {corepkgs => src/libexpr/primops}/derivation.nix (100%)

diff --git a/corepkgs/local.mk b/corepkgs/local.mk
index 2c72d3a31..57f6d53a7 100644
--- a/corepkgs/local.mk
+++ b/corepkgs/local.mk
@@ -1,6 +1,5 @@
 corepkgs_FILES = \
   unpack-channel.nix \
-  derivation.nix \
   fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index d84b150e0..687a8ccda 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -42,6 +42,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d0b0c57b2..7e8526ea1 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3565,13 +3565,11 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    try {
-        string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
-        sDerivationNix = symbols.create(path);
-        evalFile(path, v);
-        addConstant("derivation", v);
-    } catch (SysError &) {
-    }
+    sDerivationNix = symbols.create("//builtin/derivation.nix");
+    eval(parse(
+        #include "primops/derivation.nix.gen.hh"
+        , foFile, sDerivationNix, "/", staticBaseEnv), v);
+    addConstant("derivation", v);
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
diff --git a/corepkgs/derivation.nix b/src/libexpr/primops/derivation.nix
similarity index 100%
rename from corepkgs/derivation.nix
rename to src/libexpr/primops/derivation.nix

From 787469c7b66aec12ab6847e7db2cdc8aef5c325e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Sep 2020 09:35:30 +0200
Subject: [PATCH 226/882] Remove corepkgs/unpack-channel.nix

---
 corepkgs/local.mk                                |  1 -
 src/nix-channel/nix-channel.cc                   | 11 +++++++++--
 {corepkgs => src/nix-channel}/unpack-channel.nix |  0
 src/nix/local.mk                                 |  2 ++
 4 files changed, 11 insertions(+), 3 deletions(-)
 rename {corepkgs => src/nix-channel}/unpack-channel.nix (100%)

diff --git a/corepkgs/local.mk b/corepkgs/local.mk
index 57f6d53a7..1546b4ef3 100644
--- a/corepkgs/local.mk
+++ b/corepkgs/local.mk
@@ -1,5 +1,4 @@
 corepkgs_FILES = \
-  unpack-channel.nix \
   fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 3ccf620c9..e48f7af9a 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -76,6 +76,13 @@ static void update(const StringSet & channelNames)
 
     auto store = openStore();
 
+    auto [fd, unpackChannelPath] = createTempFile();
+    writeFull(fd.get(),
+        #include "unpack-channel.nix.gen.hh"
+        );
+    fd = -1;
+    AutoDelete del(unpackChannelPath, false);
+
     // Download each channel.
     Strings exprs;
     for (const auto & channel : channels) {
@@ -104,7 +111,7 @@ static void update(const StringSet & channelNames)
 
         bool unpacked = false;
         if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
-            runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
+            runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
                         "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
             unpacked = true;
         }
@@ -125,7 +132,7 @@ static void update(const StringSet & channelNames)
     // Unpack the channel tarballs into the Nix store and install them
     // into the channels profile.
     std::cerr << "unpacking channels...\n";
-    Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
+    Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" };
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));
     envArgs.push_back("--quiet");
diff --git a/corepkgs/unpack-channel.nix b/src/nix-channel/unpack-channel.nix
similarity index 100%
rename from corepkgs/unpack-channel.nix
rename to src/nix-channel/unpack-channel.nix
diff --git a/src/nix/local.mk b/src/nix/local.mk
index e96200685..ab4e9121b 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -29,3 +29,5 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
 
 src/nix/develop.cc: src/nix/get-env.sh.gen.hh
+
+src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh

From c9f51e87057652db0013289a95deffba495b35e7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Sep 2020 10:42:51 +0200
Subject: [PATCH 227/882] Remove corepkgs/config.nix

This isn't used anywhere except in the configure script of the Perl
bindings. I've changed the latter to use the C++ API's Settings object
at runtime.
---
 .gitignore                |  9 ------
 corepkgs/config.nix.in    | 13 ---------
 corepkgs/local.mk         |  4 +--
 perl/configure.ac         | 13 ---------
 perl/lib/Nix/Config.pm.in | 10 ++-----
 perl/lib/Nix/Store.pm     | 60 ++-------------------------------------
 perl/lib/Nix/Store.xs     | 10 +++++++
 tests/recursive.sh        |  6 ++--
 8 files changed, 19 insertions(+), 106 deletions(-)
 delete mode 100644 corepkgs/config.nix.in

diff --git a/.gitignore b/.gitignore
index 3f81f8ef5..4711fa7fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,15 +12,6 @@ perl/Makefile.config
 /svn-revision
 /libtool
 
-/corepkgs/config.nix
-
-# /corepkgs/channels/
-/corepkgs/channels/unpack.sh
-
-# /corepkgs/nar/
-/corepkgs/nar/nar.sh
-/corepkgs/nar/unnar.sh
-
 # /doc/manual/
 /doc/manual/*.1
 /doc/manual/*.5
diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in
deleted file mode 100644
index cb9945944..000000000
--- a/corepkgs/config.nix.in
+++ /dev/null
@@ -1,13 +0,0 @@
-# FIXME: remove this file?
-let
-  fromEnv = var: def:
-    let val = builtins.getEnv var; in
-    if val != "" then val else def;
-in rec {
-  nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
-  nixPrefix = "@prefix@";
-  nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
-  nixLocalstateDir = "@localstatedir@";
-  nixSysconfDir = "@sysconfdir@";
-  nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
-}
diff --git a/corepkgs/local.mk b/corepkgs/local.mk
index 1546b4ef3..0bc91cfab 100644
--- a/corepkgs/local.mk
+++ b/corepkgs/local.mk
@@ -1,6 +1,4 @@
 corepkgs_FILES = \
   fetchurl.nix
 
-$(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
-
-template-files += $(d)/config.nix
+$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
diff --git a/perl/configure.ac b/perl/configure.ac
index c3769e142..255744afd 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -70,19 +70,6 @@ PKG_CHECK_MODULES([NIX], [nix-store])
 
 NEED_PROG([NIX], [nix])
 
-# Get nix configure values
-export NIX_REMOTE=daemon
-nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixBinDir)
-nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLibexecDir)
-nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLocalstateDir)
-nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixSysconfDir)
-nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixStoreDir)
-AC_SUBST(nixbindir)
-AC_SUBST(nixlibexecdir)
-AC_SUBST(nixlocalstatedir)
-AC_SUBST(nixsysconfdir)
-AC_SUBST(nixstoredir)
-
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in
index bc1749e60..f7c6f2484 100644
--- a/perl/lib/Nix/Config.pm.in
+++ b/perl/lib/Nix/Config.pm.in
@@ -4,14 +4,8 @@ use MIME::Base64;
 
 $version = "@PACKAGE_VERSION@";
 
-$binDir = $ENV{"NIX_BIN_DIR"} || "@nixbindir@";
-$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@nixlibexecdir@";
-$stateDir = $ENV{"NIX_STATE_DIR"} || "@nixlocalstatedir@/nix";
-$logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
-$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
-$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
-
-$useBindings = 1;
+$binDir = Nix::Store::getBinDir;
+$storeDir = Nix::Store::getStoreDir;
 
 %config = ();
 
diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm
index d226264d4..179f1dc90 100644
--- a/perl/lib/Nix/Store.pm
+++ b/perl/lib/Nix/Store.pm
@@ -2,7 +2,6 @@ package Nix::Store;
 
 use strict;
 use warnings;
-use Nix::Config;
 
 require Exporter;
 
@@ -22,6 +21,7 @@ our @EXPORT = qw(
     addToStore makeFixedOutputPath
     derivationFromPath
     addTempRoot
+    getBinDir getStoreDir
 );
 
 our $VERSION = '0.15';
@@ -34,62 +34,8 @@ sub backtick {
     return $res;
 }
 
-if ($Nix::Config::useBindings) {
-    require XSLoader;
-    XSLoader::load('Nix::Store', $VERSION);
-} else {
-
-    # Provide slow fallbacks of some functions on platforms that don't
-    # support the Perl bindings.
-
-    use File::Temp;
-    use Fcntl qw/F_SETFD/;
-
-    *hashFile = sub {
-        my ($algo, $base32, $path) = @_;
-        my $res = backtick("$Nix::Config::binDir/nix-hash", "--flat", $path, "--type", $algo, $base32 ? "--base32" : ());
-        chomp $res;
-        return $res;
-    };
-
-    *hashPath = sub {
-        my ($algo, $base32, $path) = @_;
-        my $res = backtick("$Nix::Config::binDir/nix-hash", $path, "--type", $algo, $base32 ? "--base32" : ());
-        chomp $res;
-        return $res;
-    };
-
-    *hashString = sub {
-        my ($algo, $base32, $s) = @_;
-        my $fh = File::Temp->new();
-        print $fh $s;
-        my $res = backtick("$Nix::Config::binDir/nix-hash", $fh->filename, "--type", $algo, $base32 ? "--base32" : ());
-        chomp $res;
-        return $res;
-    };
-
-    *addToStore = sub {
-        my ($srcPath, $recursive, $algo) = @_;
-        die "not implemented" if $recursive || $algo ne "sha256";
-        my $res = backtick("$Nix::Config::binDir/nix-store", "--add", $srcPath);
-        chomp $res;
-        return $res;
-    };
-
-    *isValidPath = sub {
-        my ($path) = @_;
-        my $res = backtick("$Nix::Config::binDir/nix-store", "--check-validity", "--print-invalid", $path);
-        chomp $res;
-        return $res ne $path;
-    };
-
-    *queryPathHash = sub {
-        my ($path) = @_;
-        my $res = backtick("$Nix::Config::binDir/nix-store", "--query", "--hash", $path);
-        chomp $res;
-        return $res;
-    };
-}
+require XSLoader;
+XSLoader::load('Nix::Store', $VERSION);
 
 1;
 __END__
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 8546f6307..599921151 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -351,3 +351,13 @@ void addTempRoot(char * storePath)
         } catch (Error & e) {
             croak("%s", e.what());
         }
+
+
+SV * getBinDir()
+    PPCODE:
+        XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0)));
+
+
+SV * getStoreDir()
+    PPCODE:
+        XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0)));
diff --git a/tests/recursive.sh b/tests/recursive.sh
index cf10d55bf..80a178cc7 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -9,9 +9,8 @@ rm -f $TEST_ROOT/result
 
 export unreachable=$(nix add-to-store ./recursive.sh)
 
-nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
+NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
   with import ./config.nix;
-  with import <nix/config.nix>;
   mkDerivation {
     name = "recursive";
     dummy = builtins.toFile "dummy" "bla bla";
@@ -24,9 +23,10 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu
 
     buildCommand = '\'\''
       mkdir $out
-      PATH=${nixBinDir}:$PATH
       opts="--experimental-features nix-command"
 
+      PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
+
       # Check that we can query/build paths in our input closure.
       nix $opts path-info $dummy
       nix $opts build $dummy

From 520895b1dac1b82be9f571df3981a15cfada2968 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 17 Sep 2020 13:36:58 +0200
Subject: [PATCH 228/882] Fix garbage collection of CA derivations

Fix #4026
---
 src/libstore/gc.cc         | 9 ++++++---
 tests/content-addressed.sh | 3 +++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index e6cbc525d..08b53c702 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -574,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     /* If keep-derivations is set and this is a derivation, then
        don't delete the derivation if any of the outputs are alive. */
     if (state.gcKeepDerivations && path.isDerivation()) {
-        for (auto & i : queryDerivationOutputs(path))
-            if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                incoming.insert(i);
+        for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
+            if (maybeOutPath &&
+                isValidPath(*maybeOutPath) &&
+                queryPathInfo(*maybeOutPath)->deriver == path
+                )
+                incoming.insert(*maybeOutPath);
     }
 
     /* If keep-outputs is set, then don't delete this path if there
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index ae9e3c59e..0ae2852d2 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -15,3 +15,6 @@ out1=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 1)
 out2=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 2)
 
 test $out1 == $out2
+
+nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
+nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true

From 3f93bc0d39661af52466a316a3daf2e49165755e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 17 Sep 2020 17:38:25 +0200
Subject: [PATCH 229/882] Typo

---
 src/libutil/split.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/split.hh b/src/libutil/split.hh
index d19d7d8ed..87a23b13e 100644
--- a/src/libutil/split.hh
+++ b/src/libutil/split.hh
@@ -9,7 +9,7 @@ namespace nix {
 
 // If `separator` is found, we return the portion of the string before the
 // separator, and modify the string argument to contain only the part after the
-// separator. Otherwise, wer return `std::nullopt`, and we leave the argument
+// separator. Otherwise, we return `std::nullopt`, and we leave the argument
 // string alone.
 static inline std::optional<std::string_view> splitPrefixTo(std::string_view & string, char separator) {
     auto sepInstance = string.find(separator);

From 9ee3122ec71ae43f5cd8bf0a9282777ba17342c5 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 13 Sep 2020 14:38:05 +0200
Subject: [PATCH 230/882] Remove redundant import

---
 src/libstore/daemon.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 8adabf549..2e068992b 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -2,7 +2,6 @@
 #include "monitor-fd.hh"
 #include "worker-protocol.hh"
 #include "store-api.hh"
-#include "local-store.hh"
 #include "finally.hh"
 #include "affinity.hh"
 #include "archive.hh"

From 29c82ccc77714a74e8948ce8c531de5a8d870176 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 13 Sep 2020 14:39:11 +0200
Subject: [PATCH 231/882] Add Source.drainInto(Sink)

---
 src/libutil/serialise.cc | 13 ++++++++++---
 src/libutil/serialise.hh |  2 ++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 00c945113..a469a1e73 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -93,7 +93,7 @@ void Source::operator () (unsigned char * data, size_t len)
 }
 
 
-std::string Source::drain()
+void Source::drainInto(Sink & sink)
 {
     std::string s;
     std::vector<unsigned char> buf(8192);
@@ -101,12 +101,19 @@ std::string Source::drain()
         size_t n;
         try {
             n = read(buf.data(), buf.size());
-            s.append((char *) buf.data(), n);
+            sink(buf.data(), n);
         } catch (EndOfFile &) {
             break;
         }
     }
-    return s;
+}
+
+
+std::string Source::drain()
+{
+    StringSink s;
+    drainInto(s);
+    return *s.s;
 }
 
 
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 6f4f4c855..2fd06bb4d 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -69,6 +69,8 @@ struct Source
 
     virtual bool good() { return true; }
 
+    void drainInto(Sink & sink);
+
     std::string drain();
 };
 

From dfa547c6a81d6fb7ef3d3f69a98ebe969df42828 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 17 Sep 2020 17:15:05 +0200
Subject: [PATCH 232/882] Add ContentAddressMethod and parse/render it

---
 src/libstore/content-address.cc | 62 ++++++++++++++++++++++++++-------
 src/libstore/content-address.hh | 19 ++++++++++
 2 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 0885c3d0e..bf56d3d77 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -37,48 +37,86 @@ std::string renderContentAddress(ContentAddress ca) {
     }, ca);
 }
 
-ContentAddress parseContentAddress(std::string_view rawCa) {
-    auto rest = rawCa;
+std::string renderContentAddressMethod(ContentAddressMethod cam) {
+    return std::visit(overloaded {
+        [](TextHashMethod &th) {
+            return std::string{"text:"} + printHashType(htSHA256);
+        },
+        [](FixedOutputHashMethod &fshm) {
+            return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
+        }
+    }, cam);
+}
+
+/*
+  Parses content address strings up to the hash.
+ */
+static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) {
+    std::string wholeInput(rest);
 
     std::string_view prefix;
     {
         auto optPrefix = splitPrefixTo(rest, ':');
         if (!optPrefix)
-            throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
+            throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput);
         prefix = *optPrefix;
     }
 
     auto parseHashType_ = [&](){
         auto hashTypeRaw = splitPrefixTo(rest, ':');
         if (!hashTypeRaw)
-            throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
+            throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
         HashType hashType = parseHashType(*hashTypeRaw);
         return std::move(hashType);
     };
 
     // Switch on prefix
     if (prefix == "text") {
-        // No parsing of the method, "text" only support flat.
+        // No parsing of the ingestion method, "text" only support flat.
         HashType hashType = parseHashType_();
         if (hashType != htSHA256)
             throw Error("text content address hash should use %s, but instead uses %s",
                 printHashType(htSHA256), printHashType(hashType));
-        return TextHash {
-            .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
-        };
+        return TextHashMethod {};
     } else if (prefix == "fixed") {
         // Parse method
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
         HashType hashType = parseHashType_();
-        return FixedOutputHash {
-            .method = method,
-            .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+        return FixedOutputHashMethod {
+            .fileIngestionMethod = method,
+            .hashType = std::move(hashType),
         };
     } else
         throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
-};
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+    auto rest = rawCa;
+
+    ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
+
+    return std::visit(
+        overloaded {
+            [&](TextHashMethod thm) {
+                return ContentAddress(TextHash {
+                    .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
+                });
+            },
+            [&](FixedOutputHashMethod fohMethod) {
+                return ContentAddress(FixedOutputHash {
+                    .method = fohMethod.fileIngestionMethod,
+                    .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
+                });
+            },
+        }, caMethod);
+}
+
+ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) {
+    std::string_view asPrefix {std::string{caMethod} + ":"};
+    return parseContentAddressMethodPrefix(asPrefix);
+}
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
     return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index 22a039242..f6a6f5140 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -55,4 +55,23 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
 Hash getContentAddressHash(const ContentAddress & ca);
 
+/*
+  We only have one way to hash text with references, so this is single-value
+  type is only useful in std::variant.
+*/
+struct TextHashMethod { };
+struct FixedOutputHashMethod {
+  FileIngestionMethod fileIngestionMethod;
+  HashType hashType;
+};
+
+typedef std::variant<
+    TextHashMethod,
+    FixedOutputHashMethod
+  > ContentAddressMethod;
+
+ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
+
+std::string renderContentAddressMethod(ContentAddressMethod caMethod);
+
 }

From 14b30b3f3d5af75c210a15cb128e67c0eff66149 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 17 Sep 2020 17:36:16 +0200
Subject: [PATCH 233/882] Move FramedSource and FramedSink, extract
 withFramedSink

---
 src/libstore/daemon.cc       |  47 -----------
 src/libstore/remote-store.cc | 151 +++++++++++++++++++----------------
 src/libutil/serialise.hh     |  47 +++++++++++
 3 files changed, 127 insertions(+), 118 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 2e068992b..010d3bc2d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -748,59 +748,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             info.ultimate = false;
 
         if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
-
-            struct FramedSource : Source
-            {
-                Source & from;
-                bool eof = false;
-                std::vector<unsigned char> pending;
-                size_t pos = 0;
-
-                FramedSource(Source & from) : from(from)
-                { }
-
-                ~FramedSource()
-                {
-                    if (!eof) {
-                        while (true) {
-                            auto n = readInt(from);
-                            if (!n) break;
-                            std::vector<unsigned char> data(n);
-                            from(data.data(), n);
-                        }
-                    }
-                }
-
-                size_t read(unsigned char * data, size_t len) override
-                {
-                    if (eof) throw EndOfFile("reached end of FramedSource");
-
-                    if (pos >= pending.size()) {
-                        size_t len = readInt(from);
-                        if (!len) {
-                            eof = true;
-                            return 0;
-                        }
-                        pending = std::vector<unsigned char>(len);
-                        pos = 0;
-                        from(pending.data(), len);
-                    }
-
-                    auto n = std::min(len, pending.size() - pos);
-                    memcpy(data, pending.data() + pos, n);
-                    pos += n;
-                    return n;
-                }
-            };
-
             logger->startWork();
-
             {
                 FramedSource source(from);
                 store->addToStore(info, source, (RepairFlag) repair,
                     dontCheckSigs ? NoCheckSigs : CheckSigs);
             }
-
             logger->stopWork();
         }
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index e92b94975..993b05dc0 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -306,6 +306,8 @@ struct ConnectionHandle
             std::rethrow_exception(ex);
         }
     }
+
+    void withFramedSink(std::function<void(Sink & sink)> fun);
 };
 
 
@@ -564,78 +566,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << repair << !checkSigs;
 
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
-
-            conn->to.flush();
-
-            std::exception_ptr ex;
-
-            struct FramedSink : BufferedSink
-            {
-                ConnectionHandle & conn;
-                std::exception_ptr & ex;
-
-                FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
-                { }
-
-                ~FramedSink()
-                {
-                    try {
-                        conn->to << 0;
-                        conn->to.flush();
-                    } catch (...) {
-                        ignoreException();
-                    }
-                }
-
-                void write(const unsigned char * data, size_t len) override
-                {
-                    /* Don't send more data if the remote has
-                       encountered an error. */
-                    if (ex) {
-                        auto ex2 = ex;
-                        ex = nullptr;
-                        std::rethrow_exception(ex2);
-                    }
-                    conn->to << len;
-                    conn->to(data, len);
-                };
-            };
-
-            /* Handle log messages / exceptions from the remote on a
-               separate thread. */
-            std::thread stderrThread([&]()
-            {
-                try {
-                    conn.processStderr(nullptr, nullptr, false);
-                } catch (...) {
-                    ex = std::current_exception();
-                }
-            });
-
-            Finally joinStderrThread([&]()
-            {
-                if (stderrThread.joinable()) {
-                    stderrThread.join();
-                    if (ex) {
-                        try {
-                            std::rethrow_exception(ex);
-                        } catch (...) {
-                            ignoreException();
-                        }
-                    }
-                }
-            });
-
-            {
-                FramedSink sink(conn, ex);
+            conn.withFramedSink([&](Sink & sink) {
                 copyNAR(source, sink);
-                sink.flush();
-            }
-
-            stderrThread.join();
-            if (ex)
-                std::rethrow_exception(ex);
-
+            });
         } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
             conn.processStderr(0, &source);
         } else {
@@ -992,6 +925,82 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
+
+struct FramedSink : nix::BufferedSink
+{
+    ConnectionHandle & conn;
+    std::exception_ptr & ex;
+
+    FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
+    { }
+
+    ~FramedSink()
+    {
+        try {
+            conn->to << 0;
+            conn->to.flush();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        /* Don't send more data if the remote has
+            encountered an error. */
+        if (ex) {
+            auto ex2 = ex;
+            ex = nullptr;
+            std::rethrow_exception(ex2);
+        }
+        conn->to << len;
+        conn->to(data, len);
+    };
+};
+
+void
+ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) {
+    (*this)->to.flush();
+
+    std::exception_ptr ex;
+
+    /* Handle log messages / exceptions from the remote on a
+        separate thread. */
+    std::thread stderrThread([&]()
+    {
+        try {
+            processStderr(nullptr, nullptr, false);
+        } catch (...) {
+            ex = std::current_exception();
+        }
+    });
+
+    Finally joinStderrThread([&]()
+    {
+        if (stderrThread.joinable()) {
+            stderrThread.join();
+            if (ex) {
+                try {
+                    std::rethrow_exception(ex);
+                } catch (...) {
+                    ignoreException();
+                }
+            }
+        }
+    });
+
+    {
+        FramedSink sink(*this, ex);
+        fun(sink);
+        sink.flush();
+    }
+
+    stderrThread.join();
+    if (ex)
+        std::rethrow_exception(ex);
+
+}
+
 static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
 
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 2fd06bb4d..6027d5961 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -406,4 +406,51 @@ struct StreamToSourceAdapter : Source
 };
 
 
+/* Like SizedSource, but guarantees that the whole frame is consumed after
+   destruction. This ensures that the original stream is in a known state. */
+struct FramedSource : Source
+{
+    Source & from;
+    bool eof = false;
+    std::vector<unsigned char> pending;
+    size_t pos = 0;
+
+    FramedSource(Source & from) : from(from)
+    { }
+
+    ~FramedSource()
+    {
+        if (!eof) {
+            while (true) {
+                auto n = readInt(from);
+                if (!n) break;
+                std::vector<unsigned char> data(n);
+                from(data.data(), n);
+            }
+        }
+    }
+
+    size_t read(unsigned char * data, size_t len) override
+    {
+        if (eof) throw EndOfFile("reached end of FramedSource");
+
+        if (pos >= pending.size()) {
+            size_t len = readInt(from);
+            if (!len) {
+                eof = true;
+                return 0;
+            }
+            pending = std::vector<unsigned char>(len);
+            pos = 0;
+            from(pending.data(), len);
+        }
+
+        auto n = std::min(len, pending.size() - pos);
+        memcpy(data, pending.data() + pos, n);
+        pos += n;
+        return n;
+    }
+};
+
+
 }

From 958bf5712377f59622c59f05a84641aa1093fd32 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Sep 2020 13:10:42 +0200
Subject: [PATCH 234/882] nix build: find() -> get()

find() returns an iterator so "!attr" doesn't work.
---
 src/nix/bundle.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 241c8699b..fc41da9e4 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -98,14 +98,14 @@ struct CmdBundle : InstallableCommand
         if (!evalState->isDerivation(*vRes))
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());
 
-        auto attr1 = vRes->attrs->find(evalState->sDrvPath);
+        auto attr1 = vRes->attrs->get(evalState->sDrvPath);
         if (!attr1)
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());
 
         PathSet context2;
         StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
 
-        auto attr2 = vRes->attrs->find(evalState->sOutPath);
+        auto attr2 = vRes->attrs->get(evalState->sOutPath);
         if (!attr2)
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());
 

From 2bcf8cbe7a36ebcd2ad27b4df18fdb911d7ab224 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 18 Sep 2020 14:10:45 +0200
Subject: [PATCH 235/882] libfetchers/github: allow `url` attribute

Since 108debef6f703aa3ca1f6d6c45428880624ec6f8 we allow a
`url`-attribute for the `github`-fetcher to fetch tarballs from
self-hosted `gitlab`/`github` instances.

However it's not used when defining e.g. a flake-input

    foobar = {
        type = "github";
        url = "gitlab.myserver";
        /* ... */
    }

and breaks with an evaluation-error:

    error: --- Error --------------------------------------nix
    unsupported input attribute 'url'
    (use '--show-trace' to show detailed location information)

This patch allows flake-inputs to be fetched from self-hosted instances
as well.
---
 src/libfetchers/github.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 1cc0c5e2e..f510ae5f5 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -77,7 +77,7 @@ struct GitArchiveInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != type()) return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "url")
                 throw Error("unsupported input attribute '%s'", name);
 
         getStrAttr(attrs, "owner");

From 5fe375a8f1d60b49835b52df48686caddaa297a4 Mon Sep 17 00:00:00 2001
From: Bryan Richter <bryan.richter@relexsolutions.com>
Date: Fri, 18 Sep 2020 18:36:17 +0300
Subject: [PATCH 236/882] nix-prefetch-url: Add --executable flag

pkgs.fetchurl supports an executable argument, which is especially nice
when downloading a large executable. This patch adds the same option to
nix-prefetch-url.

I have tested this to work on the simple case of prefetching a little
executable:

1. nix-prefetch-url --executable https://my/little/script
2. Paste the hash into a pkgs.fetchurl-based package, script-pkg.nix
3. Delete the output from the store to avoid any misidentified artifacts
4. Realise the package script-pkg.nix
5. Run the executable

I repeated the above while using --name, as well.

I suspect --executable would have no meaningful effect if combined with
--unpack, but I have not tried it.
---
 doc/manual/src/command-ref/nix-prefetch-url.md |  3 +++
 src/nix-prefetch-url/nix-prefetch-url.cc       | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index 1cd1063cd..1307c7c37 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -51,6 +51,9 @@ Nix store is also printed.
     result to the Nix store. The resulting hash can be used with
     functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`.
 
+  - `--executable`  
+    Set the executable bit on the downloaded file.
+
   - `--name` *name*  
     Override the name of the file in the Nix store. By default, this is
     `hash-basename`, where *basename* is the last component of *url*.
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 1001f27af..377ae03a8 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -57,6 +57,7 @@ static int _main(int argc, char * * argv)
         bool fromExpr = false;
         string attrPath;
         bool unpack = false;
+        bool executable = false;
         string name;
 
         struct MyArgs : LegacyArgs, MixEvalArgs
@@ -81,6 +82,8 @@ static int _main(int argc, char * * argv)
             }
             else if (*arg == "--unpack")
                 unpack = true;
+            else if (*arg == "--executable")
+                executable = true;
             else if (*arg == "--name")
                 name = getArg(*arg, arg, end);
             else if (*arg != "" && arg->at(0) == '-')
@@ -175,7 +178,11 @@ static int _main(int argc, char * * argv)
 
             /* Download the file. */
             {
-                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+                auto mode = 0600;
+                if (executable)
+                    mode = 0700;
+
+                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
                 if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
 
                 FdSink sink(fd.get());
@@ -201,7 +208,7 @@ static int _main(int argc, char * * argv)
                     tmpFile = unpacked;
             }
 
-            const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
 
             auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
             storePath = info.path;

From c00e07834327a8ef626cf4f1ecb216ee1b6a0877 Mon Sep 17 00:00:00 2001
From: Marwan Aljubeh <marwan.aljubeh@gmail.com>
Date: Fri, 18 Sep 2020 17:10:39 +0100
Subject: [PATCH 237/882] Add a nix.conf option for allowing a symlinked store

---
 src/libstore/globals.cc     |  1 +
 src/libstore/globals.hh     | 13 +++++++++++++
 src/libstore/local-store.cc |  2 +-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 491c664db..ca2e75603 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -41,6 +41,7 @@ Settings::Settings()
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
+    ignoreSymlinkStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
 
     caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
     if (caFile == "") {
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 02721285a..129cef6b4 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -881,6 +881,19 @@ public:
 
     Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
         "Path or URI of the global flake registry."};
+
+    Setting<bool> ignoreSymlinkStore{
+        this, false, "ignore-symlink-store",
+        R"(
+	  If set to `true`, Nix will stop complaining if the store directory
+	  (typically /nix/store) contains symlink components.
+
+	  This risks making some builds "impure" because builders sometimes
+	  "canonicalise" paths by resolving all symlink components. Problems
+          occur if those builds are then deployed to machines where /nix/store
+          resolves to a different location from that of the build machine. You
+          can enable this setting if you are sure you're not going to do that.
+        )"};
 };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c618203f0..24b9ea7bd 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -109,7 +109,7 @@ LocalStore::LocalStore(const Params & params)
     }
 
     /* Ensure that the store and its parents are not symlinks. */
-    if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") {
+    if (!settings.ignoreSymlinkStore) {
         Path path = realStoreDir;
         struct stat st;
         while (path != "/") {

From e40772cd35adcd158d30727f7f294b823df8010a Mon Sep 17 00:00:00 2001
From: Marwan Aljubeh <marwan.aljubeh@gmail.com>
Date: Fri, 18 Sep 2020 17:18:45 +0100
Subject: [PATCH 238/882] Lint issue: replacing tabs with spaces

---
 src/libstore/globals.hh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 129cef6b4..ddc13898d 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -885,11 +885,11 @@ public:
     Setting<bool> ignoreSymlinkStore{
         this, false, "ignore-symlink-store",
         R"(
-	  If set to `true`, Nix will stop complaining if the store directory
-	  (typically /nix/store) contains symlink components.
+          If set to `true`, Nix will stop complaining if the store directory
+          (typically /nix/store) contains symlink components.
 
-	  This risks making some builds "impure" because builders sometimes
-	  "canonicalise" paths by resolving all symlink components. Problems
+          This risks making some builds "impure" because builders sometimes
+          "canonicalise" paths by resolving all symlink components. Problems
           occur if those builds are then deployed to machines where /nix/store
           resolves to a different location from that of the build machine. You
           can enable this setting if you are sure you're not going to do that.

From e34fe47d0ce19fc7657970fb0e610bffbc3e43f0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 17 Sep 2020 19:27:11 +0200
Subject: [PATCH 239/882] Overhaul wopAddToStore

---
 src/libstore/daemon.cc          | 105 ++++++++++++++++++++------------
 src/libstore/remote-store.cc    | 103 +++++++++++++++++++------------
 src/libstore/remote-store.hh    |   7 +--
 src/libstore/worker-protocol.hh |   2 +-
 4 files changed, 133 insertions(+), 84 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 010d3bc2d..0a08a2e12 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -349,47 +349,74 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddToStore: {
-        HashType hashAlgo;
-        std::string baseName;
-        FileIngestionMethod method;
-        {
-            bool fixed;
-            uint8_t recursive;
-            std::string hashAlgoRaw;
-            from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
-            if (recursive > (uint8_t) FileIngestionMethod::Recursive)
-                throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
-            method = FileIngestionMethod { recursive };
-            /* Compatibility hack. */
-            if (!fixed) {
-                hashAlgoRaw = "sha256";
-                method = FileIngestionMethod::Recursive;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
+            auto name = readString(from);
+            auto camStr = readString(from);
+            auto refs = readStorePaths<StorePathSet>(*store, from);
+
+            logger->startWork();
+            StorePath path = [&]() -> StorePath {
+                // NB: FramedSource must be out of scope before logger->stopWork();
+                ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
+                FramedSource source(from);
+                return std::visit(overloaded {
+                    [&](TextHashMethod &_) -> StorePath {
+                        // We could stream this by changing Store
+                        std::string contents = source.drain();
+                        return store->addTextToStore(name, contents, refs);
+                    },
+                    [&](FixedOutputHashMethod &fohm) -> StorePath {
+                        return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType);
+                    },
+                }, contentAddressMethod);
+            }();
+            logger->stopWork();
+
+            to << store->printStorePath(path);
+
+        } else {
+            HashType hashAlgo;
+            std::string baseName;
+            FileIngestionMethod method;
+            {
+                bool fixed;
+                uint8_t recursive;
+                std::string hashAlgoRaw;
+                from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
+                if (recursive > (uint8_t) FileIngestionMethod::Recursive)
+                    throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
+                method = FileIngestionMethod { recursive };
+                /* Compatibility hack. */
+                if (!fixed) {
+                    hashAlgoRaw = "sha256";
+                    method = FileIngestionMethod::Recursive;
+                }
+                hashAlgo = parseHashType(hashAlgoRaw);
             }
-            hashAlgo = parseHashType(hashAlgoRaw);
+
+            StringSink saved;
+            TeeSource savedNARSource(from, saved);
+            RetrieveRegularNARSink savedRegular { saved };
+
+            if (method == FileIngestionMethod::Recursive) {
+                /* Get the entire NAR dump from the client and save it to
+                  a string so that we can pass it to
+                  addToStoreFromDump(). */
+                ParseSink sink; /* null sink; just parse the NAR */
+                parseDump(sink, savedNARSource);
+            } else
+                parseDump(savedRegular, from);
+
+            logger->startWork();
+            if (!savedRegular.regular) throw Error("regular file expected");
+
+            // FIXME: try to stream directly from `from`.
+            StringSource dumpSource { *saved.s };
+            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
+            logger->stopWork();
+
+            to << store->printStorePath(path);
         }
-
-        StringSink saved;
-        TeeSource savedNARSource(from, saved);
-        RetrieveRegularNARSink savedRegular { saved };
-
-        if (method == FileIngestionMethod::Recursive) {
-            /* Get the entire NAR dump from the client and save it to
-               a string so that we can pass it to
-               addToStoreFromDump(). */
-            ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNARSource);
-        } else
-            parseDump(savedRegular, from);
-
-        logger->startWork();
-        if (!savedRegular.regular) throw Error("regular file expected");
-
-        // FIXME: try to stream directly from `from`.
-        StringSource dumpSource { *saved.s };
-        auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
-        logger->stopWork();
-
-        to << store->printStorePath(path);
         break;
     }
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 993b05dc0..14ad4767d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -526,6 +526,69 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 }
 
 
+StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashType, RepairFlag repair)
+{
+    if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon");
+    StorePathSet references;
+
+    auto conn(getConnection());
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
+
+        conn->to
+            << wopAddToStore
+            << name
+            << renderContentAddressMethod(FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType });
+        writeStorePaths(*this, conn->to, references);
+
+        conn.withFramedSink([&](Sink & sink) {
+            dump.drainInto(sink);
+        });
+
+        return parseStorePath(readString(conn->from));
+
+    }
+    else {
+        if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
+
+        conn->to
+            << wopAddToStore
+            << name
+            << ((hashType == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+            << (method == FileIngestionMethod::Recursive ? 1 : 0)
+            << printHashType(hashType);
+
+        try {
+            conn->to.written = 0;
+            conn->to.warn = true;
+            connections->incCapacity();
+            {
+                Finally cleanup([&]() { connections->decCapacity(); });
+                if (method == FileIngestionMethod::Recursive) {
+                    dump.drainInto(conn->to);
+                } else {
+                    std::string contents = dump.drain();
+                    dumpString(contents, conn->to);
+                }
+            }
+            conn->to.warn = false;
+            conn.processStderr();
+        } catch (SysError & e) {
+            /* Daemon closed while we were sending the path. Probably OOM
+              or I/O error. */
+            if (e.errNo == EPIPE)
+                try {
+                    conn.processStderr();
+                } catch (EndOfFile & e) { }
+            throw;
+        }
+
+        return parseStorePath(readString(conn->from));
+    }
+}
+
+
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
@@ -579,46 +642,6 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
-{
-    if (repair) throw Error("repairing is not supported when building through the Nix daemon");
-
-    auto conn(getConnection());
-
-    Path srcPath(absPath(_srcPath));
-
-    conn->to
-        << wopAddToStore
-        << name
-        << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-        << (method == FileIngestionMethod::Recursive ? 1 : 0)
-        << printHashType(hashAlgo);
-
-    try {
-        conn->to.written = 0;
-        conn->to.warn = true;
-        connections->incCapacity();
-        {
-            Finally cleanup([&]() { connections->decCapacity(); });
-            dumpPath(srcPath, conn->to, filter);
-        }
-        conn->to.warn = false;
-        conn.processStderr();
-    } catch (SysError & e) {
-        /* Daemon closed while we were sending the path. Probably OOM
-           or I/O error. */
-        if (e.errNo == EPIPE)
-            try {
-                conn.processStderr();
-            } catch (EndOfFile & e) { }
-        throw;
-    }
-
-    return parseStorePath(readString(conn->from));
-}
-
-
 StorePath RemoteStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 91c748006..14c22e764 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -63,13 +63,12 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
-
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 13cf8d4ab..21e05d91c 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x118
+#define PROTOCOL_VERSION 0x119
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

From c602ebfb34de3626fa0b9110face6ea4b171ac0f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 17 Sep 2020 20:19:15 +0200
Subject: [PATCH 240/882] Refactor wopAddToStore to make wopAddTextToStore
 obsolete

---
 src/libstore/remote-store.cc    | 91 ++++++++++++++++++---------------
 src/libstore/remote-store.hh    |  2 +
 src/libstore/worker-protocol.hh |  2 +-
 3 files changed, 53 insertions(+), 42 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 14ad4767d..95e6a3291 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -526,12 +526,8 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 }
 
 
-StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method, HashType hashType, RepairFlag repair)
+StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references)
 {
-    if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon");
-    StorePathSet references;
-
     auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
@@ -539,7 +535,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
         conn->to
             << wopAddToStore
             << name
-            << renderContentAddressMethod(FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType });
+            << renderContentAddressMethod(caMethod);
         writeStorePaths(*this, conn->to, references);
 
         conn.withFramedSink([&](Sink & sink) {
@@ -552,42 +548,60 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
     else {
         if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
 
-        conn->to
-            << wopAddToStore
-            << name
-            << ((hashType == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-            << (method == FileIngestionMethod::Recursive ? 1 : 0)
-            << printHashType(hashType);
+        std::visit(overloaded {
+            [&](TextHashMethod thm) -> void {
+                std::string s = dump.drain();
+                conn->to << wopAddTextToStore << name << s;
+                writeStorePaths(*this, conn->to, references);
+                conn.processStderr();
+            },
+            [&](FixedOutputHashMethod fohm) -> void {
+                conn->to
+                    << wopAddToStore
+                    << name
+                    << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
+                    << printHashType(fohm.hashType);
 
-        try {
-            conn->to.written = 0;
-            conn->to.warn = true;
-            connections->incCapacity();
-            {
-                Finally cleanup([&]() { connections->decCapacity(); });
-                if (method == FileIngestionMethod::Recursive) {
-                    dump.drainInto(conn->to);
-                } else {
-                    std::string contents = dump.drain();
-                    dumpString(contents, conn->to);
-                }
-            }
-            conn->to.warn = false;
-            conn.processStderr();
-        } catch (SysError & e) {
-            /* Daemon closed while we were sending the path. Probably OOM
-              or I/O error. */
-            if (e.errNo == EPIPE)
                 try {
+                    conn->to.written = 0;
+                    conn->to.warn = true;
+                    connections->incCapacity();
+                    {
+                        Finally cleanup([&]() { connections->decCapacity(); });
+                        if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
+                            dump.drainInto(conn->to);
+                        } else {
+                            std::string contents = dump.drain();
+                            dumpString(contents, conn->to);
+                        }
+                    }
+                    conn->to.warn = false;
                     conn.processStderr();
-                } catch (EndOfFile & e) { }
-            throw;
-        }
+                } catch (SysError & e) {
+                    /* Daemon closed while we were sending the path. Probably OOM
+                      or I/O error. */
+                    if (e.errNo == EPIPE)
+                        try {
+                            conn.processStderr();
+                        } catch (EndOfFile & e) { }
+                    throw;
+                }
 
+            }
+        }, caMethod);
         return parseStorePath(readString(conn->from));
     }
 }
 
+StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashType, RepairFlag repair)
+{
+    if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon");
+    StorePathSet references;
+    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references);
+}
+
 
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
@@ -646,13 +660,8 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
-
-    auto conn(getConnection());
-    conn->to << wopAddTextToStore << name << s;
-    writeStorePaths(*this, conn->to, references);
-
-    conn.processStderr();
-    return parseStorePath(readString(conn->from));
+    StringSource source(s);
+    return addCAToStore(source, name, TextHashMethod{}, references);
 }
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 14c22e764..2c775cb79 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -63,6 +63,8 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
+    StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references);
+
     StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 21e05d91c..b100d1550 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -18,7 +18,7 @@ typedef enum {
     wopQueryReferences = 5, // obsolete
     wopQueryReferrers = 6,
     wopAddToStore = 7,
-    wopAddTextToStore = 8,
+    wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore
     wopBuildPaths = 9,
     wopEnsurePath = 10,
     wopAddTempRoot = 11,

From ecc8088cb7e91e4c45787f290c8ff61547fb838a Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Thu, 17 Sep 2020 21:32:01 +0200
Subject: [PATCH 241/882] wopAddToStore: Throw to clarify unused refs

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/libstore/daemon.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 0a08a2e12..f2d789699 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -366,6 +366,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                         return store->addTextToStore(name, contents, refs);
                     },
                     [&](FixedOutputHashMethod &fohm) -> StorePath {
+                        if (!refs.empty())
+                            throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
                         return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType);
                     },
                 }, contentAddressMethod);

From 8279178b078103f816bf85f5a153a4845d30cc83 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 17 Sep 2020 22:01:35 +0200
Subject: [PATCH 242/882] Move FramedSink next to FramedSource

---
 src/libstore/remote-store.cc | 35 +--------------------------
 src/libutil/serialise.hh     | 46 ++++++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 36 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 95e6a3291..cfbf23ac4 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -957,39 +957,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-
-struct FramedSink : nix::BufferedSink
-{
-    ConnectionHandle & conn;
-    std::exception_ptr & ex;
-
-    FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
-    { }
-
-    ~FramedSink()
-    {
-        try {
-            conn->to << 0;
-            conn->to.flush();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    void write(const unsigned char * data, size_t len) override
-    {
-        /* Don't send more data if the remote has
-            encountered an error. */
-        if (ex) {
-            auto ex2 = ex;
-            ex = nullptr;
-            std::rethrow_exception(ex2);
-        }
-        conn->to << len;
-        conn->to(data, len);
-    };
-};
-
 void
 ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) {
     (*this)->to.flush();
@@ -1022,7 +989,7 @@ ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) {
     });
 
     {
-        FramedSink sink(*this, ex);
+        FramedSink sink((*this)->to, ex);
         fun(sink);
         sink.flush();
     }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 6027d5961..3511e4cd8 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -406,8 +406,13 @@ struct StreamToSourceAdapter : Source
 };
 
 
-/* Like SizedSource, but guarantees that the whole frame is consumed after
-   destruction. This ensures that the original stream is in a known state. */
+/* A source that reads a distinct format of concatenated chunks back into its
+   logical form, in order to guarantee a known state to the original stream,
+   even in the event of errors.
+
+   Use with FramedSink, which also allows the logical stream to be terminated
+   in the event of an exception.
+*/
 struct FramedSource : Source
 {
     Source & from;
@@ -452,5 +457,42 @@ struct FramedSource : Source
     }
 };
 
+/* Write as chunks in the format expected by FramedSource.
+
+   The exception_ptr reference can be used to terminate the stream when you
+   detect that an error has occurred on the remote end.
+*/
+struct FramedSink : nix::BufferedSink
+{
+    BufferedSink & to;
+    std::exception_ptr & ex;
+
+    FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex)
+    { }
+
+    ~FramedSink()
+    {
+        try {
+            to << 0;
+            to.flush();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        /* Don't send more data if the remote has
+            encountered an error. */
+        if (ex) {
+            auto ex2 = ex;
+            ex = nullptr;
+            std::rethrow_exception(ex2);
+        }
+        to << len;
+        to(data, len);
+    };
+};
+
 
 }

From fbf509c1137fd59ebb3e7a993c8da42c17deb68a Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Thu, 17 Sep 2020 22:04:05 +0200
Subject: [PATCH 243/882] parseContentAddressMethodPrefix: use string_view

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/libstore/content-address.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index bf56d3d77..74215c545 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -52,7 +52,7 @@ std::string renderContentAddressMethod(ContentAddressMethod cam) {
   Parses content address strings up to the hash.
  */
 static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) {
-    std::string wholeInput(rest);
+    std::string_view wholeInput { rest };
 
     std::string_view prefix;
     {

From 7c682640857106f18d7020c9c75ea39b1ef8dd2c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 18 Sep 2020 10:06:34 +0200
Subject: [PATCH 244/882] wopAddToStore: add RepairFlag

---
 src/libstore/daemon.cc       | 5 ++++-
 src/libstore/remote-store.cc | 9 ++++-----
 src/libstore/remote-store.hh | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index f2d789699..9f7350217 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -353,6 +353,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             auto name = readString(from);
             auto camStr = readString(from);
             auto refs = readStorePaths<StorePathSet>(*store, from);
+            bool repairBool;
+            from >> repairBool;
+            auto repair = RepairFlag{repairBool};
 
             logger->startWork();
             StorePath path = [&]() -> StorePath {
@@ -368,7 +371,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     [&](FixedOutputHashMethod &fohm) -> StorePath {
                         if (!refs.empty())
                             throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
-                        return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType);
+                        return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
                     },
                 }, contentAddressMethod);
             }();
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index cfbf23ac4..54d70e3e1 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -526,7 +526,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 }
 
 
-StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references)
+StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair)
 {
     auto conn(getConnection());
 
@@ -537,6 +537,7 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA
             << name
             << renderContentAddressMethod(caMethod);
         writeStorePaths(*this, conn->to, references);
+        conn->to << repair;
 
         conn.withFramedSink([&](Sink & sink) {
             dump.drainInto(sink);
@@ -597,9 +598,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA
 StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method, HashType hashType, RepairFlag repair)
 {
-    if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon");
     StorePathSet references;
-    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references);
+    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair);
 }
 
 
@@ -659,9 +659,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 StorePath RemoteStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
-    if (repair) throw Error("repairing is not supported when building through the Nix daemon");
     StringSource source(s);
-    return addCAToStore(source, name, TextHashMethod{}, references);
+    return addCAToStore(source, name, TextHashMethod{}, references, repair);
 }
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 2c775cb79..23f7b8425 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -63,7 +63,7 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
-    StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references);
+    StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair);
 
     StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;

From fa08db5c4c92ed4674cfacd131d1b45906a9b146 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 18 Sep 2020 11:22:13 +0200
Subject: [PATCH 245/882] wopAddToStore: return ValidPathInfo

A ValidPathInfo is created anyway. By returning it we can save a
roundtrip and we have a nicer interface.
---
 src/libstore/daemon.cc       | 38 ++++++++++++++++++-------------
 src/libstore/remote-store.cc | 43 +++++++++++++++++++++---------------
 src/libstore/remote-store.hh |  4 +++-
 3 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 9f7350217..8897a73f4 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -239,6 +239,18 @@ struct ClientSettings
     }
 };
 
+static void writeValidPathInfo(ref<Store> store, unsigned int clientVersion, Sink & to, std::shared_ptr<const ValidPathInfo> info) {
+    to << (info->deriver ? store->printStorePath(*info->deriver) : "")
+       << info->narHash.to_string(Base16, false);
+    writeStorePaths(*store, to, info->references);
+    to << info->registrationTime << info->narSize;
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
+        to << info->ultimate
+           << info->sigs
+           << renderContentAddress(info->ca);
+    }
+}
+
 static void performOp(TunnelLogger * logger, ref<Store> store,
     TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
     Source & from, BufferedSink & to, unsigned int op)
@@ -358,26 +370,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             auto repair = RepairFlag{repairBool};
 
             logger->startWork();
-            StorePath path = [&]() -> StorePath {
+            auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
                 ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
                 FramedSource source(from);
+                // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
                 return std::visit(overloaded {
-                    [&](TextHashMethod &_) -> StorePath {
+                    [&](TextHashMethod &_) {
                         // We could stream this by changing Store
                         std::string contents = source.drain();
-                        return store->addTextToStore(name, contents, refs);
+                        auto path = store->addTextToStore(name, contents, refs, repair);
+                        return store->queryPathInfo(path);
                     },
-                    [&](FixedOutputHashMethod &fohm) -> StorePath {
+                    [&](FixedOutputHashMethod &fohm) {
                         if (!refs.empty())
                             throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
-                        return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
+                        auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
+                        return store->queryPathInfo(path);
                     },
                 }, contentAddressMethod);
             }();
             logger->stopWork();
 
-            to << store->printStorePath(path);
+            to << store->printStorePath(pathInfo->path);
+            writeValidPathInfo(store, clientVersion, to, pathInfo);
 
         } else {
             HashType hashAlgo;
@@ -706,15 +722,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (info) {
             if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
                 to << 1;
-            to << (info->deriver ? store->printStorePath(*info->deriver) : "")
-               << info->narHash.to_string(Base16, false);
-            writeStorePaths(*store, to, info->references);
-            to << info->registrationTime << info->narSize;
-            if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
-                to << info->ultimate
-                   << info->sigs
-                   << renderContentAddress(info->ca);
-            }
+            writeValidPathInfo(store, clientVersion, to, info);
         } else {
             assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
             to << 0;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 54d70e3e1..3936ddb1d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -421,11 +421,27 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
 }
 
 
+ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) {
+    auto deriver = readString(conn->from);
+    auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
+    auto info = make_ref<ValidPathInfo>(path, narHash);
+    if (deriver != "") info->deriver = parseStorePath(deriver);
+    info->references = readStorePaths<StorePathSet>(*this, conn->from);
+    conn->from >> info->registrationTime >> info->narSize;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
+        conn->from >> info->ultimate;
+        info->sigs = readStrings<StringSet>(conn->from);
+        info->ca = parseContentAddressOpt(readString(conn->from));
+    }
+    return info;
+}
+
+
 void RemoteStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        std::shared_ptr<ValidPathInfo> info;
+        std::shared_ptr<const ValidPathInfo> info;
         {
             auto conn(getConnection());
             conn->to << wopQueryPathInfo << printStorePath(path);
@@ -441,17 +457,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
                 bool valid; conn->from >> valid;
                 if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
             }
-            auto deriver = readString(conn->from);
-            auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
-            info = std::make_shared<ValidPathInfo>(path, narHash);
-            if (deriver != "") info->deriver = parseStorePath(deriver);
-            info->references = readStorePaths<StorePathSet>(*this, conn->from);
-            conn->from >> info->registrationTime >> info->narSize;
-            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-                conn->from >> info->ultimate;
-                info->sigs = readStrings<StringSet>(conn->from);
-                info->ca = parseContentAddressOpt(readString(conn->from));
-            }
+            info = readValidPathInfo(conn, path);
         }
         callback(std::move(info));
     } catch (...) { callback.rethrow(); }
@@ -526,7 +532,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 }
 
 
-StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair)
+ref<const ValidPathInfo> RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair)
 {
     auto conn(getConnection());
 
@@ -543,8 +549,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA
             dump.drainInto(sink);
         });
 
-        return parseStorePath(readString(conn->from));
-
+        auto path = parseStorePath(readString(conn->from));
+        return readValidPathInfo(conn, path);
     }
     else {
         if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
@@ -591,7 +597,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA
 
             }
         }, caMethod);
-        return parseStorePath(readString(conn->from));
+        auto path = parseStorePath(readString(conn->from));
+        return queryPathInfo(path);
     }
 }
 
@@ -599,7 +606,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method, HashType hashType, RepairFlag repair)
 {
     StorePathSet references;
-    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair);
+    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
 }
 
 
@@ -660,7 +667,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
     StringSource source(s);
-    return addCAToStore(source, name, TextHashMethod{}, references, repair);
+    return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
 }
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 23f7b8425..47f43ae8a 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -63,7 +63,7 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
-    StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair);
+    ref<const ValidPathInfo> addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair);
 
     StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
@@ -140,6 +140,8 @@ protected:
 
     virtual void narFromPath(const StorePath & path, Sink & sink) override;
 
+    ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
+
 private:
 
     std::atomic_bool failed{false};

From ca30abb3fb36440e5a13161c39647189036fc18f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 18 Sep 2020 11:58:45 +0200
Subject: [PATCH 246/882] Document addCAToStore/addToStoreFromDump source
 drainage

Also checked that all usages satisfy the requirement and
removed dead code.
---
 src/libstore/build.cc        | 8 --------
 src/libstore/remote-store.hh | 2 ++
 src/libstore/store-api.hh    | 3 ++-
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6e55f83d5..061f07546 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2949,14 +2949,6 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
         goal.addDependency(info.path);
     }
 
-    StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair = NoRepair) override
     {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 47f43ae8a..735a3c24e 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -63,8 +63,10 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
+    /* Add a content-addressable store path. `dump` will be drained. */
     ref<const ValidPathInfo> addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair);
 
+    /* Add a content-addressable store path. Does not support references. `dump` will be drained. */
     StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 4d3f07dfc..591140874 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -449,7 +449,8 @@ public:
     /* Like addToStore(), but the contents of the path are contained
        in `dump', which is either a NAR serialisation (if recursive ==
        true) or simply the contents of a regular file (if recursive ==
-       false). */
+       false).
+       `dump` may be drained */
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)

From 9aa0dafe205899f350382b587e284c493b223cdd Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 21 Sep 2020 13:11:31 +0200
Subject: [PATCH 247/882] Update lowdown version

Fix #4042

According to https://github.com/kristapsdz/lowdown/commit/8aef9e9290de22a10c14ae138257bc1c7fa8ba1f, we shouldn't need to use a fork anymore so we can switch back to upstream
---
 flake.lock | 11 +++++------
 flake.nix  |  2 +-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/flake.lock b/flake.lock
index f4368b170..822a73332 100644
--- a/flake.lock
+++ b/flake.lock
@@ -3,16 +3,15 @@
     "lowdown-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1598296217,
-        "narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=",
-        "owner": "edolstra",
+        "lastModified": 1598695561,
+        "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=",
+        "owner": "kristapsdz",
         "repo": "lowdown",
-        "rev": "c7a4e715af1e233080842db82d15b261cb74cb28",
+        "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
-        "ref": "no-structs-in-anonymous-unions",
+        "owner": "kristapsdz",
         "repo": "lowdown",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index a50533a29..1b9eb4c77 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
-  inputs.lowdown-src = { url = "github:edolstra/lowdown/no-structs-in-anonymous-unions"; flake = false; };
+  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
   outputs = { self, nixpkgs, lowdown-src }:
 

From d110fdd03f8860b2a1cd689187f8056b9e22af09 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 13:28:51 +0200
Subject: [PATCH 248/882] Disable precompiled headers in 'nix develop'

They're still enabled in regular builds though.
---
 flake.nix                 | 5 +----
 mk/precompiled-headers.mk | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index 1b9eb4c77..0304557e8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -136,7 +136,7 @@
 
           enableParallelBuilding = true;
 
-          makeFlags = "profiledir=$(out)/etc/profile.d";
+          makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
 
           doCheck = true;
 
@@ -334,9 +334,6 @@
             # syntax-check generated dot files, it still requires some
             # fonts.  So provide those.
             FONTCONFIG_FILE = texFunctions.fontsConf;
-
-            # To test building without precompiled headers.
-            makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
           };
 
         # System tests.
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index 500c99e4a..1fdb4b3a4 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -1,4 +1,4 @@
-PRECOMPILE_HEADERS ?= 1
+PRECOMPILE_HEADERS ?= 0
 
 print-var-help += \
   echo "  PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build";

From 56f1e0df0546c744421c7d1bd31b1a341796513c Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 21 Sep 2020 16:29:08 +0200
Subject: [PATCH 249/882] libfetchers/github: rename `url` to `host`

---
 src/libfetchers/github.cc | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index f510ae5f5..a4db5c5fa 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -8,9 +8,9 @@
 
 namespace nix::fetchers {
 
-// A github or gitlab url
-const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
-std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
+// A github or gitlab host
+const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme
 {
@@ -50,9 +50,9 @@ struct GitArchiveInputScheme : InputScheme
                     throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
                 ref = value;
             }
-            else if (name == "url") {
-                if (!std::regex_match(value, urlRegex))
-                    throw BadURL("URL '%s' contains an invalid instance url", url.url);
+            else if (name == "host") {
+                if (!std::regex_match(value, hostRegex))
+                    throw BadURL("URL '%s' contains an invalid instance host", url.url);
                 host_url = value;
             }
             // FIXME: barf on unsupported attributes
@@ -67,7 +67,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("repo", path[1]);
         if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
         if (ref) input.attrs.insert_or_assign("ref", *ref);
-        if (host_url) input.attrs.insert_or_assign("url", *host_url);
+        if (host_url) input.attrs.insert_or_assign("host", *host_url);
 
         return input;
     }
@@ -77,7 +77,7 @@ struct GitArchiveInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != type()) return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "url")
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
                 throw Error("unsupported input attribute '%s'", name);
 
         getStrAttr(attrs, "owner");
@@ -210,7 +210,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
@@ -236,7 +236,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
         auto json = nlohmann::json::parse(

From 4e1a04733d5075fdc09dbc6767755d4487e96da7 Mon Sep 17 00:00:00 2001
From: Marwan Aljubeh <marwan.aljubeh@gmail.com>
Date: Mon, 21 Sep 2020 16:32:22 +0100
Subject: [PATCH 250/882] Use a better name for the config option

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index ddc13898d..fcb9b0f63 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -883,7 +883,7 @@ public:
         "Path or URI of the global flake registry."};
 
     Setting<bool> ignoreSymlinkStore{
-        this, false, "ignore-symlink-store",
+        this, false, "allow-symlinked-store",
         R"(
           If set to `true`, Nix will stop complaining if the store directory
           (typically /nix/store) contains symlink components.

From e8e1d420f364afbfface61d3f03889e10e6066c9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 18:22:45 +0200
Subject: [PATCH 251/882] Don't include <regex> in header files

This reduces compilation time by ~15 seconds (CPU time).

Issue #4045.
---
 src/libexpr/eval.cc                   |  1 +
 src/libexpr/eval.hh                   |  8 +++--
 src/libexpr/flake/flakeref.cc         |  1 +
 src/libexpr/flake/lockfile.cc         |  1 +
 src/libexpr/primops.cc                | 18 ++++++++---
 src/libexpr/primops/fetchMercurial.cc |  3 +-
 src/libfetchers/git.cc                |  1 +
 src/libfetchers/github.cc             |  1 +
 src/libfetchers/indirect.cc           |  1 +
 src/libfetchers/mercurial.cc          |  1 +
 src/libstore/names.cc                 | 21 +++++++++++--
 src/libstore/names.hh                 |  7 +++--
 src/libutil/url-parts.hh              | 44 +++++++++++++++++++++++++++
 src/libutil/url.cc                    |  1 +
 src/libutil/url.hh                    | 38 -----------------------
 src/nix-env/nix-env.cc                |  2 +-
 16 files changed, 96 insertions(+), 53 deletions(-)
 create mode 100644 src/libutil/url-parts.hh

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 139067f20..883fc27a7 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -356,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sEpsilon(symbols.create(""))
     , repair(NoRepair)
     , store(store)
+    , regexCache(makeRegexCache())
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
 {
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 80078d8a5..0e1f61baa 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -6,7 +6,6 @@
 #include "symbol-table.hh"
 #include "config.hh"
 
-#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
@@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath;
 void initGC();
 
 
+struct RegexCache;
+
+std::shared_ptr<RegexCache> makeRegexCache();
+
+
 class EvalState
 {
 public:
@@ -120,7 +124,7 @@ private:
     std::unordered_map<Path, Path> resolvedPaths;
 
     /* Cache used by prim_match(). */
-    std::unordered_map<std::string, std::regex> regexCache;
+    std::shared_ptr<RegexCache> regexCache;
 
 public:
 
diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
index 6363446f6..d5c2ffe66 100644
--- a/src/libexpr/flake/flakeref.cc
+++ b/src/libexpr/flake/flakeref.cc
@@ -1,6 +1,7 @@
 #include "flakeref.hh"
 #include "store-api.hh"
 #include "url.hh"
+#include "url-parts.hh"
 #include "fetchers.hh"
 #include "registry.hh"
 
diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index a74846944..78431f000 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -1,5 +1,6 @@
 #include "lockfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7e8526ea1..9cfe3f402 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3085,17 +3085,25 @@ static RegisterPrimOp primop_hashString({
     .fun = prim_hashString,
 });
 
-/* Match a regular expression against a string and return either
-   ‘null’ or a list containing substring matches. */
+struct RegexCache
+{
+    std::unordered_map<std::string, std::regex> cache;
+};
+
+std::shared_ptr<RegexCache> makeRegexCache()
+{
+    return std::make_shared<RegexCache>();
+}
+
 void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        auto regex = state.regexCache.find(re);
-        if (regex == state.regexCache.end())
-            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
+        auto regex = state.regexCache->cache.find(re);
+        if (regex == state.regexCache->cache.end())
+            regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first;
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index cef85cfef..1a064ed5c 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -3,8 +3,7 @@
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "url.hh"
-
-#include <regex>
+#include "url-parts.hh"
 
 namespace nix {
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 5ca0f8521..ad7638d73 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <sys/time.h>
 
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index a4db5c5fa..1737658a7 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -3,6 +3,7 @@
 #include "fetchers.hh"
 #include "globals.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index b981d4d8e..74332ae3d 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -1,4 +1,5 @@
 #include "fetchers.hh"
+#include "url-parts.hh"
 
 namespace nix::fetchers {
 
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 3e76ffc4d..d80c2ea7a 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <sys/time.h>
 
diff --git a/src/libstore/names.cc b/src/libstore/names.cc
index d1c8a6101..41e28dc99 100644
--- a/src/libstore/names.cc
+++ b/src/libstore/names.cc
@@ -1,10 +1,18 @@
 #include "names.hh"
 #include "util.hh"
 
+#include <regex>
+
 
 namespace nix {
 
 
+struct Regex
+{
+    std::regex regex;
+};
+
+
 DrvName::DrvName()
 {
     name = "";
@@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0)
 }
 
 
+DrvName::~DrvName()
+{ }
+
+
 bool DrvName::matches(DrvName & n)
 {
     if (name != "*") {
-        if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
-        if (!std::regex_match(n.name, *regex)) return false;
+        if (!regex) {
+            regex = std::make_unique<Regex>();
+            regex->regex = std::regex(name, std::regex::extended);
+        }
+        if (!std::regex_match(n.name, regex->regex)) return false;
     }
     if (version != "" && version != n.version) return false;
     return true;
@@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;
     for (auto & i : opArgs)
-        result.push_back(DrvName(i));
+        result.emplace_back(i);
     return result;
 }
 
diff --git a/src/libstore/names.hh b/src/libstore/names.hh
index 00e14b8c7..bc62aac93 100644
--- a/src/libstore/names.hh
+++ b/src/libstore/names.hh
@@ -3,10 +3,11 @@
 #include <memory>
 
 #include "types.hh"
-#include <regex>
 
 namespace nix {
 
+struct Regex;
+
 struct DrvName
 {
     string fullName;
@@ -16,10 +17,12 @@ struct DrvName
 
     DrvName();
     DrvName(std::string_view s);
+    ~DrvName();
+
     bool matches(DrvName & n);
 
 private:
-    std::unique_ptr<std::regex> regex;
+    std::unique_ptr<Regex> regex;
 };
 
 typedef list<DrvName> DrvNames;
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
new file mode 100644
index 000000000..64e06cfbc
--- /dev/null
+++ b/src/libutil/url-parts.hh
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <string>
+#include <regex>
+
+namespace nix {
+
+// URI stuff.
+const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
+const static std::string schemeRegex = "(?:[a-z+.-]+)";
+const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
+const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
+const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
+const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
+const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
+const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
+const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
+const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
+const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
+const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
+const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
+const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
+
+// A Git ref (i.e. branch or tag name).
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+extern std::regex refRegex;
+
+// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
+// This is because of the definition of a ref in refs.c in https://github.com/git/git
+// See tests/fetchGitRefs.sh for the full definition
+const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
+extern std::regex badGitRefRegex;
+
+// A Git revision (a SHA-1 commit hash).
+const static std::string revRegexS = "[0-9a-fA-F]{40}";
+extern std::regex revRegex;
+
+// A ref or revision, or a ref followed by a revision.
+const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
+
+const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
+extern std::regex flakeIdRegex;
+
+}
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 88c09eef9..c1bab866c 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -1,4 +1,5 @@
 #include "url.hh"
+#include "url-parts.hh"
 #include "util.hh"
 
 namespace nix {
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 1f716ba10..6e77142e3 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -2,8 +2,6 @@
 
 #include "error.hh"
 
-#include <regex>
-
 namespace nix {
 
 struct ParsedURL
@@ -29,40 +27,4 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
 
 ParsedURL parseURL(const std::string & url);
 
-// URI stuff.
-const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
-const static std::string schemeRegex = "(?:[a-z+.-]+)";
-const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
-const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
-const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
-const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
-const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
-const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
-const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
-const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
-const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
-const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
-const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
-const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
-
-// A Git ref (i.e. branch or tag name).
-const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
-extern std::regex refRegex;
-
-// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
-// This is because of the definition of a ref in refs.c in https://github.com/git/git
-// See tests/fetchGitRefs.sh for the full definition
-const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
-extern std::regex badGitRefRegex;
-
-// A Git revision (a SHA-1 commit hash).
-const static std::string revRegexS = "[0-9a-fA-F]{40}";
-extern std::regex revRegex;
-
-// A ref or revision, or a ref followed by a revision.
-const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
-
-const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
-extern std::regex flakeIdRegex;
-
 }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index e5a433ac0..3e7c453fb 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
 {
     DrvNames selectors = drvNamesFromArgs(args);
     if (selectors.empty())
-        selectors.push_back(DrvName("*"));
+        selectors.emplace_back("*");
 
     DrvInfos elems;
     set<unsigned int> done;

From f80ffeb8c9291f7168f098fdaadc15408492f3c2 Mon Sep 17 00:00:00 2001
From: Marwan Aljubeh <marwan.aljubeh@gmail.com>
Date: Mon, 21 Sep 2020 17:29:08 +0100
Subject: [PATCH 252/882] Update the variable name accordingly

---
 src/libstore/globals.cc     | 2 +-
 src/libstore/globals.hh     | 2 +-
 src/libstore/local-store.cc | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index ca2e75603..ff6082aac 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -41,7 +41,7 @@ Settings::Settings()
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
-    ignoreSymlinkStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
+    allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
 
     caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
     if (caFile == "") {
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index fcb9b0f63..fd0c6cbcc 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -882,7 +882,7 @@ public:
     Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
         "Path or URI of the global flake registry."};
 
-    Setting<bool> ignoreSymlinkStore{
+    Setting<bool> allowSymlinkedStore{
         this, false, "allow-symlinked-store",
         R"(
           If set to `true`, Nix will stop complaining if the store directory
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 24b9ea7bd..cc9fcfe6e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -109,7 +109,7 @@ LocalStore::LocalStore(const Params & params)
     }
 
     /* Ensure that the store and its parents are not symlinks. */
-    if (!settings.ignoreSymlinkStore) {
+    if (!settings.allowSymlinkedStore) {
         Path path = realStoreDir;
         struct stat st;
         while (path != "/") {

From d51ba430473368b29abfd1a8c4655da74b3a780c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 18:40:11 +0200
Subject: [PATCH 253/882] Move Callback into its own header

This gets rid of the inclusion of <future> in util.hh, cutting
compilation time by ~20s (CPU time).

Issue #4045.
---
 src/libstore/binary-cache-store.cc      |  1 +
 src/libstore/build.cc                   |  1 +
 src/libstore/dummy-store.cc             |  1 +
 src/libstore/filetransfer.cc            |  1 +
 src/libstore/http-binary-cache-store.cc |  1 +
 src/libstore/legacy-ssh-store.cc        |  1 +
 src/libstore/local-store.cc             |  1 +
 src/libstore/misc.cc                    |  2 +-
 src/libstore/remote-store.cc            |  1 +
 src/libstore/store-api.cc               |  4 +--
 src/libutil/callback.hh                 | 46 +++++++++++++++++++++++++
 src/libutil/util.hh                     | 38 +-------------------
 12 files changed, 57 insertions(+), 41 deletions(-)
 create mode 100644 src/libutil/callback.hh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 34f844a18..ebc0bd6a4 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -11,6 +11,7 @@
 #include "nar-accessor.hh"
 #include "json.hh"
 #include "thread-pool.hh"
+#include "callback.hh"
 
 #include <chrono>
 #include <future>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6e55f83d5..0b51d90ea 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -17,6 +17,7 @@
 #include "daemon.hh"
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
+#include "callback.hh"
 
 #include <algorithm>
 #include <iostream>
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 128832e60..49641c2ac 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -1,4 +1,5 @@
 #include "store-api.hh"
+#include "callback.hh"
 
 namespace nix {
 
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 4149f8155..6241b5e00 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -5,6 +5,7 @@
 #include "s3.hh"
 #include "compression.hh"
 #include "finally.hh"
+#include "callback.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index f4ab15a10..86be7c006 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -2,6 +2,7 @@
 #include "filetransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
+#include "callback.hh"
 
 namespace nix {
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index e9478c1d5..5af75669a 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -6,6 +6,7 @@
 #include "worker-protocol.hh"
 #include "ssh.hh"
 #include "derivations.hh"
+#include "callback.hh"
 
 namespace nix {
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c618203f0..94ff34cb8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -6,6 +6,7 @@
 #include "derivations.hh"
 #include "nar-info.hh"
 #include "references.hh"
+#include "callback.hh"
 
 #include <iostream>
 #include <algorithm>
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index da3981696..ad4dccef9 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -5,7 +5,7 @@
 #include "store-api.hh"
 #include "thread-pool.hh"
 #include "topo-sort.hh"
-
+#include "callback.hh"
 
 namespace nix {
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index e92b94975..27535f1d0 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -10,6 +10,7 @@
 #include "pool.hh"
 #include "finally.hh"
 #include "logging.hh"
+#include "callback.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 2d5077ed0..1bbc74db8 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -8,9 +8,7 @@
 #include "json.hh"
 #include "url.hh"
 #include "archive.hh"
-
-#include <future>
-
+#include "callback.hh"
 
 namespace nix {
 
diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh
new file mode 100644
index 000000000..ef31794be
--- /dev/null
+++ b/src/libutil/callback.hh
@@ -0,0 +1,46 @@
+#pragma once
+
+#include <future>
+#include <functional>
+
+namespace nix {
+
+/* A callback is a wrapper around a lambda that accepts a valid of
+   type T or an exception. (We abuse std::future<T> to pass the value or
+   exception.) */
+template<typename T>
+class Callback
+{
+    std::function<void(std::future<T>)> fun;
+    std::atomic_flag done = ATOMIC_FLAG_INIT;
+
+public:
+
+    Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
+
+    Callback(Callback && callback) : fun(std::move(callback.fun))
+    {
+        auto prev = callback.done.test_and_set();
+        if (prev) done.test_and_set();
+    }
+
+    void operator()(T && t) noexcept
+    {
+        auto prev = done.test_and_set();
+        assert(!prev);
+        std::promise<T> promise;
+        promise.set_value(std::move(t));
+        fun(promise.get_future());
+    }
+
+    void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
+    {
+        auto prev = done.test_and_set();
+        assert(!prev);
+        std::promise<T> promise;
+        promise.set_exception(exc);
+        fun(promise.get_future());
+    }
+};
+
+}
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 082e26375..91e0a1543 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -17,7 +17,6 @@
 #include <map>
 #include <sstream>
 #include <optional>
-#include <future>
 #include <iterator>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
@@ -480,43 +479,8 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
 }
 
 
-/* A callback is a wrapper around a lambda that accepts a valid of
-   type T or an exception. (We abuse std::future<T> to pass the value or
-   exception.) */
 template<typename T>
-class Callback
-{
-    std::function<void(std::future<T>)> fun;
-    std::atomic_flag done = ATOMIC_FLAG_INIT;
-
-public:
-
-    Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
-
-    Callback(Callback && callback) : fun(std::move(callback.fun))
-    {
-        auto prev = callback.done.test_and_set();
-        if (prev) done.test_and_set();
-    }
-
-    void operator()(T && t) noexcept
-    {
-        auto prev = done.test_and_set();
-        assert(!prev);
-        std::promise<T> promise;
-        promise.set_value(std::move(t));
-        fun(promise.get_future());
-    }
-
-    void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
-    {
-        auto prev = done.test_and_set();
-        assert(!prev);
-        std::promise<T> promise;
-        promise.set_exception(exc);
-        fun(promise.get_future());
-    }
-};
+class Callback;
 
 
 /* Start a thread that handles various signals. Also block those signals

From 340ca382c4de5863f0ecb5c67bae5461ea20c60b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 18:47:18 +0200
Subject: [PATCH 254/882] Don't include nlohmann/json.hpp in globals.hh

This reduces compilation time by 207s.

Issue #4045.
---
 src/libstore/globals.cc | 1 +
 src/libstore/globals.hh | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 491c664db..7b841d925 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "archive.hh"
 #include "args.hh"
+#include "abstractsettingtojson.hh"
 
 #include <algorithm>
 #include <map>
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 02721285a..8a2d3ff75 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -2,7 +2,6 @@
 
 #include "types.hh"
 #include "config.hh"
-#include "abstractsettingtojson.hh"
 #include "util.hh"
 
 #include <map>

From 0716adaa8b23855bd5e1a59f4115ec933b6a9205 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 18:49:43 +0200
Subject: [PATCH 255/882] abstractsettingtojson.hh ->
 abstract-setting-to-json.hh

---
 src/libstore/globals.cc                                         | 2 +-
 .../{abstractsettingtojson.hh => abstract-setting-to-json.hh}   | 0
 src/libutil/config.cc                                           | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename src/libutil/{abstractsettingtojson.hh => abstract-setting-to-json.hh} (100%)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 7b841d925..0beb9b2b7 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -2,7 +2,7 @@
 #include "util.hh"
 #include "archive.hh"
 #include "args.hh"
-#include "abstractsettingtojson.hh"
+#include "abstract-setting-to-json.hh"
 
 #include <algorithm>
 #include <map>
diff --git a/src/libutil/abstractsettingtojson.hh b/src/libutil/abstract-setting-to-json.hh
similarity index 100%
rename from src/libutil/abstractsettingtojson.hh
rename to src/libutil/abstract-setting-to-json.hh
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 309d23b40..5e6a211df 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -1,6 +1,6 @@
 #include "config.hh"
 #include "args.hh"
-#include "abstractsettingtojson.hh"
+#include "abstract-setting-to-json.hh"
 
 #include <nlohmann/json.hpp>
 

From 557d2427ee7c21b32e691e1c44f682b667673a70 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 18:57:02 +0200
Subject: [PATCH 256/882] Random header cleanup

---
 src/libutil/util.hh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 91e0a1543..b8e201203 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -12,12 +12,9 @@
 #include <signal.h>
 
 #include <functional>
-#include <limits>
-#include <cstdio>
 #include <map>
 #include <sstream>
 #include <optional>
-#include <iterator>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0

From ecc8672aa007af045d77434b495ca09541e9fee3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Sep 2020 19:06:23 +0200
Subject: [PATCH 257/882] fmt.hh: Don't include
 boost/algorithm/string/replace.hpp

This cuts compilation time by ~49s.

Issue #4045.
---
 src/libutil/fmt.hh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index a39de041f..6e69bdce2 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -1,7 +1,6 @@
 #pragma once
 
 #include <boost/format.hpp>
-#include <boost/algorithm/string/replace.hpp>
 #include <string>
 #include "ansicolor.hh"
 

From ba37299a0364c4b58d59e20cf7821ed9fade6dd6 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Sat, 19 Sep 2020 22:26:07 -0700
Subject: [PATCH 258/882] Serialize SandboxMode enum to string for JSON

Rather than showing an integer as the default, instead show the boolean
referenced in the description.

The nix.conf.5 manpage used to show "default: 0", which is unnecessarily
opaque and confusing (doesn't 0 mean false, even though the default is
true?); now it properly shows that the default is true.
---
 src/libstore/globals.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 0beb9b2b7..f94d97cc8 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -147,6 +147,12 @@ bool Settings::isWSL1()
 
 const string nixVersion = PACKAGE_VERSION;
 
+NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
+    {SandboxMode::smEnabled, true},
+    {SandboxMode::smRelaxed, "relaxed"},
+    {SandboxMode::smDisabled, false},
+});
+
 template<> void BaseSetting<SandboxMode>::set(const std::string & str)
 {
     if (str == "true") value = smEnabled;

From d860295e116f9aa0f37e2637b5ec4b9c86fdf4d3 Mon Sep 17 00:00:00 2001
From: Michael Reilly <OmnipotentEntity@gmail.com>
Date: Sat, 19 Sep 2020 10:46:02 -0400
Subject: [PATCH 259/882] Bump nlohmann-json version to 3.9.1

---
 src/libexpr/json-to-value.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 76e1a26bf..9ca5ac86d 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -115,6 +115,14 @@ public:
     {
         return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
+#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
+    bool binary(binary_t&)
+    {
+        // This function ought to be unreachable
+        assert(false);
+        return true;
+    }
+#endif
 
     bool start_object(std::size_t len)
     {

From 97b5154750d911a05992b97d7404d813d924de73 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Sep 2020 10:04:25 +0200
Subject: [PATCH 260/882] Disable `FORTIFY_SOURCE` when compiling without
 optims

Otherwise the build is cluttered with

```
/nix/store/fwpn2f7a4iqszyydw7ag61zlnp6xk5d3-glibc-2.30-dev/include/features.h:382:4: warning: #warning _FORTIFY_SOURCE requires compiling with optimization (-O) [-Wcpp]
  382 | #  warning _FORTIFY_SOURCE requires compiling with optimization (-O)
      |    ^~~~~~~
```

when building with `OPTIMIZE=0`
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f472ca7e5..c50d2c40f 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,7 @@ OPTIMIZE = 1
 ifeq ($(OPTIMIZE), 1)
   GLOBAL_CXXFLAGS += -O3
 else
-  GLOBAL_CXXFLAGS += -O0
+  GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
 endif
 
 include mk/lib.mk

From c1e79f870cf548e81d7cab0d296c6e923da9fa8c Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Sep 2020 10:37:43 +0200
Subject: [PATCH 261/882] Silence a compiler warning in serialise.hh

Explicitely cast to `uint64_t` in `readNum` to avoid a "comparison
between signed and unsigned" warning
---
 src/libutil/serialise.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 6f4f4c855..7682a0f19 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -340,7 +340,7 @@ T readNum(Source & source)
         ((uint64_t) buf[6] << 48) |
         ((uint64_t) buf[7] << 56);
 
-    if (n > std::numeric_limits<T>::max())
+    if (n > (uint64_t)std::numeric_limits<T>::max())
         throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
 
     return (T) n;

From 35a0ac183858ecb03e313e088562c84fe211e20d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Sep 2020 11:40:19 +0200
Subject: [PATCH 262/882] Style fixes

---
 src/libstore/content-address.cc | 26 +++++++++++++++++---------
 src/libstore/daemon.cc          |  7 ++++++-
 src/libstore/remote-store.cc    | 15 +++++++++++----
 src/libstore/remote-store.hh    |  7 ++++++-
 4 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 74215c545..90a3ad1f5 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -4,11 +4,13 @@
 
 namespace nix {
 
-std::string FixedOutputHash::printMethodAlgo() const {
+std::string FixedOutputHash::printMethodAlgo() const
+{
     return makeFileIngestionPrefix(method) + printHashType(hash.type);
 }
 
-std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+std::string makeFileIngestionPrefix(const FileIngestionMethod m)
+{
     switch (m) {
     case FileIngestionMethod::Flat:
         return "";
@@ -26,7 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
         + hash.to_string(Base32, true);
 }
 
-std::string renderContentAddress(ContentAddress ca) {
+std::string renderContentAddress(ContentAddress ca)
+{
     return std::visit(overloaded {
         [](TextHash th) {
             return "text:" + th.hash.to_string(Base32, true);
@@ -37,7 +40,8 @@ std::string renderContentAddress(ContentAddress ca) {
     }, ca);
 }
 
-std::string renderContentAddressMethod(ContentAddressMethod cam) {
+std::string renderContentAddressMethod(ContentAddressMethod cam)
+{
     return std::visit(overloaded {
         [](TextHashMethod &th) {
             return std::string{"text:"} + printHashType(htSHA256);
@@ -51,7 +55,8 @@ std::string renderContentAddressMethod(ContentAddressMethod cam) {
 /*
   Parses content address strings up to the hash.
  */
-static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) {
+static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
+{
     std::string_view wholeInput { rest };
 
     std::string_view prefix;
@@ -113,16 +118,19 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
         }, caMethod);
 }
 
-ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) {
+ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
+{
     std::string_view asPrefix {std::string{caMethod} + ":"};
     return parseContentAddressMethodPrefix(asPrefix);
 }
 
-std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
-    return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
+{
+    return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt);
 };
 
-std::string renderContentAddress(std::optional<ContentAddress> ca) {
+std::string renderContentAddress(std::optional<ContentAddress> ca)
+{
     return ca ? renderContentAddress(*ca) : "";
 }
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 8897a73f4..83f8968b0 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -239,7 +239,12 @@ struct ClientSettings
     }
 };
 
-static void writeValidPathInfo(ref<Store> store, unsigned int clientVersion, Sink & to, std::shared_ptr<const ValidPathInfo> info) {
+static void writeValidPathInfo(
+    ref<Store> store,
+    unsigned int clientVersion,
+    Sink & to,
+    std::shared_ptr<const ValidPathInfo> info)
+{
     to << (info->deriver ? store->printStorePath(*info->deriver) : "")
        << info->narHash.to_string(Base16, false);
     writeStorePaths(*store, to, info->references);
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 02a8b72b9..6f1f9769b 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -422,7 +422,8 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
 }
 
 
-ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) {
+ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
+{
     auto deriver = readString(conn->from);
     auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
     auto info = make_ref<ValidPathInfo>(path, narHash);
@@ -533,7 +534,12 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 }
 
 
-ref<const ValidPathInfo> RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair)
+ref<const ValidPathInfo> RemoteStore::addCAToStore(
+    Source & dump,
+    const string & name,
+    ContentAddressMethod caMethod,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     auto conn(getConnection());
 
@@ -603,6 +609,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(Source & dump, const string &
     }
 }
 
+
 StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method, HashType hashType, RepairFlag repair)
 {
@@ -964,8 +971,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-void
-ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) {
+void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
+{
     (*this)->to.flush();
 
     std::exception_ptr ex;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 735a3c24e..ec04be985 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -64,7 +64,12 @@ public:
         SubstitutablePathInfos & infos) override;
 
     /* Add a content-addressable store path. `dump` will be drained. */
-    ref<const ValidPathInfo> addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair);
+    ref<const ValidPathInfo> addCAToStore(
+        Source & dump,
+        const string & name,
+        ContentAddressMethod caMethod,
+        const StorePathSet & references,
+        RepairFlag repair);
 
     /* Add a content-addressable store path. Does not support references. `dump` will be drained. */
     StorePath addToStoreFromDump(Source & dump, const string & name,

From 980edd1f3a31eefe297d073f6a7cff099f21eb4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Sep 2020 15:28:20 +0200
Subject: [PATCH 263/882] RemoteStore::addCAToStore(): Don't hold connection
 while calling queryPathInfo()

This leads to a deadlock if we're at the connection limit.
---
 src/libstore/remote-store.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 6f1f9769b..be5eb4736 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -541,7 +541,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
     const StorePathSet & references,
     RepairFlag repair)
 {
-    auto conn(getConnection());
+    std::optional<ConnectionHandle> conn_(getConnection());
+    auto & conn = *conn_;
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
 
@@ -605,6 +606,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
             }
         }, caMethod);
         auto path = parseStorePath(readString(conn->from));
+        // Release our connection to prevent a deadlock in queryPathInfo().
+        conn_.reset();
         return queryPathInfo(path);
     }
 }

From 993229cdaf2e2347a204c876ecd660fc94048101 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 22 Aug 2020 20:44:47 +0000
Subject: [PATCH 264/882] Deduplicate basic derivation goals too

See comments for security concerns.

Also optimize goal creation by not traversing map twice.
---
 src/libstore/build.cc     | 90 +++++++++++++++++++++++++++------------
 src/libstore/daemon.cc    | 14 ++++++
 src/libstore/store-api.hh | 36 ++++++++++++++--
 3 files changed, 110 insertions(+), 30 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 07c5bceb2..8b206e819 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -296,9 +296,21 @@ public:
     ~Worker();
 
     /* Make a goal (with caching). */
-    GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
-        const BasicDerivation & drv, BuildMode buildMode = bmNormal);
+
+    /* derivation goal */
+private:
+    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+        const StorePath & drvPath, const StringSet & wantedOutputs,
+        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
+public:
+    std::shared_ptr<DerivationGoal> makeDerivationGoal(
+        const StorePath & drvPath,
+        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+        const StorePath & drvPath, const BasicDerivation & drv,
+        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+
+    /* substitution goal */
     GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
 
     /* Remove a dead goal. */
@@ -949,10 +961,12 @@ private:
     friend struct RestrictedStore;
 
 public:
-    DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs,
-        Worker & worker, BuildMode buildMode = bmNormal);
+    DerivationGoal(const StorePath & drvPath,
+        const StringSet & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
     DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        Worker & worker, BuildMode buildMode = bmNormal);
+        const StringSet & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
     ~DerivationGoal();
 
     /* Whether we need to perform hash rewriting if there are valid output paths. */
@@ -1085,8 +1099,8 @@ private:
 const Path DerivationGoal::homeDir = "/homeless-shelter";
 
 
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs,
-    Worker & worker, BuildMode buildMode)
+DerivationGoal::DerivationGoal(const StorePath & drvPath,
+    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker)
     , useDerivation(true)
     , drvPath(drvPath)
@@ -1094,7 +1108,9 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want
     , buildMode(buildMode)
 {
     state = &DerivationGoal::getDerivation;
-    name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath));
+    name = fmt(
+        "building of '%s' from .drv file",
+        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1103,15 +1119,18 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want
 
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    Worker & worker, BuildMode buildMode)
+    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker)
     , useDerivation(false)
     , drvPath(drvPath)
+    , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
     this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
     state = &DerivationGoal::haveDerivation;
-    name = fmt("building of %s", StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
+    name = fmt(
+        "building of '%s' from in-memory derivation",
+        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -5060,35 +5079,52 @@ Worker::~Worker()
 }
 
 
-GoalPtr Worker::makeDerivationGoal(const StorePath & path,
-    const StringSet & wantedOutputs, BuildMode buildMode)
+std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
+    const StorePath & drvPath,
+    const StringSet & wantedOutputs,
+    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
 {
-    GoalPtr goal = derivationGoals[path].lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode);
-        derivationGoals.insert_or_assign(path, goal);
+    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
+    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
+    std::shared_ptr<DerivationGoal> goal;
+    if (!abstract_goal) {
+        goal = mkDrvGoal();
+        abstract_goal_weak = goal;
         wakeUp(goal);
-    } else
-        (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
+    } else {
+        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
+        assert(goal);
+        goal->addWantedOutputs(wantedOutputs);
+    }
     return goal;
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, BuildMode buildMode)
+std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
+    const StringSet & wantedOutputs, BuildMode buildMode)
 {
-    auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode);
-    wakeUp(goal);
-    return goal;
+    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
+        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+    });
+}
+
+
+std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
+    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
+{
+    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
+        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+    });
 }
 
 
 GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
 {
-    GoalPtr goal = substitutionGoals[path].lock(); // FIXME
+    WeakGoalPtr & goal_weak = substitutionGoals[path];
+    GoalPtr goal = goal_weak.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        substitutionGoals.insert_or_assign(path, goal);
+        goal_weak = goal;
         wakeUp(goal);
     }
     return goal;
@@ -5519,7 +5555,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe
     BuildMode buildMode)
 {
     Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode);
+    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
 
     BuildResult result;
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 83f8968b0..ec3391a6d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -546,6 +546,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
            are in fact content-addressed if we don't trust them. */
         assert(derivationIsCA(drv.type()) || trusted);
 
+        /* Recompute the derivation path when we cannot trust the original. */
+        if (!trusted) {
+            /* Recomputing the derivation path for input-address derivations
+               makes it harder to audit them after the fact, since we need the
+               original not-necessarily-resolved derivation to verify the drv
+               derivation as adequate claim to the input-addressed output
+               paths. */
+            assert(derivationIsCA(drv.type()));
+
+            Derivation drv2;
+            static_cast<BasicDerivation &>(drv2) = drv;
+            drvPath = writeDerivation(*store, Derivation { drv2 });
+        }
+
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
         to << res.status << res.errorMsg;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 591140874..3ccee4f75 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -479,8 +479,38 @@ public:
         BuildMode buildMode = bmNormal);
 
     /* Build a single non-materialized derivation (i.e. not from an
-       on-disk .drv file). Note that ‘drvPath’ is only used for
-       informational purposes. */
+       on-disk .drv file).
+
+       ‘drvPath’ is used to deduplicate worker goals so it is imperative that
+       is correct. That said, it doesn't literally need to be store path that
+       would be calculated from writing this derivation to the store: it is OK
+       if it instead is that of a Derivation which would resolve to this (by
+       taking the outputs of it's input derivations and adding them as input
+       sources) such that the build time referenceable-paths are the same.
+
+       In the input-addressed case, we usually *do* use an "original"
+       unresolved derivations's path, as that is what will be used in the
+       `buildPaths` case. Also, the input-addressed output paths are verified
+       only by that contents of that specific unresolved derivation, so it is
+       nice to keep that information around so if the original derivation is
+       ever obtained later, it can be verified whether the trusted user in fact
+       used the proper output path.
+
+       In the content-addressed case, we want to always use the
+       resolved drv path calculated from the provided derivation. This serves
+       two purposes:
+
+         - It keeps the operation trustless, by ruling out a maliciously
+           invalid drv path corresponding to a non-resolution-equivalent
+           derivation.
+
+         - For the floating case in particular, it ensures that the derivation
+           to output mapping respects the resolution equivalence relation, so
+           one cannot choose different resolution-equivalent derivations to
+           subvert dependency coherence (i.e. the property that one doesn't end
+           up with multiple different versions of dependencies without
+           explicitly choosing to allow it).
+    */
     virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode = bmNormal) = 0;
 
@@ -517,7 +547,7 @@ public:
        - The collector isn't running, or it's just started but hasn't
          acquired the GC lock yet.  In that case we get and release
          the lock right away, then exit.  The collector scans the
-         permanent root and sees our's.
+         permanent root and sees ours.
 
        In either case the permanent root is seen by the collector. */
     virtual void syncWithGC() { };

From a2f5c921d41c941315ce783f66469bfb20e2c1b3 Mon Sep 17 00:00:00 2001
From: ujjwal <ujjwal.jain@vanderbilt>
Date: Tue, 22 Sep 2020 23:37:06 +0530
Subject: [PATCH 265/882] fixed typo

---
 src/libexpr/flake/flake.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 01f464859..783b98ef0 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -367,7 +367,7 @@ LockedFlake lockFlake(
 
                 /* If we have an --update-input flag for an input
                    of this input, then we must fetch the flake to
-                   to update it. */
+                   update it. */
                 auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
                 auto hasChildUpdate =

From 9fbc31a65bab50cd60a882517b3c8030485ce096 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 15 Aug 2020 16:41:28 +0000
Subject: [PATCH 266/882] Get rid of Hash::dummy from BinaryCacheStore

---
 src/libstore/binary-cache-store.cc | 102 +++++++++++++++++------------
 src/libstore/binary-cache-store.hh |   7 ++
 src/libstore/store-api.hh          |   4 +-
 3 files changed, 69 insertions(+), 44 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ebc0bd6a4..6679eb16f 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -142,17 +142,10 @@ struct FileSource : FdSource
     }
 };
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs)
+StorePath BinaryCacheStore::addToStoreCommon(
+    Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
+    std::function<ValidPathInfo(HashResult)> mkInfo)
 {
-    assert(info.narSize);
-
-    if (!repair && isValidPath(info.path)) {
-        // FIXME: copyNAR -> null sink
-        narSource.drain();
-        return;
-    }
-
     auto [fdTemp, fnTemp] = createTempFile();
 
     AutoDelete autoDelete(fnTemp);
@@ -162,13 +155,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     /* Read the NAR simultaneously into a CompressionSink+FileSink (to
        write the compressed NAR to disk), into a HashSink (to get the
        NAR hash), and into a NarAccessor (to get the NAR listing). */
-    HashSink fileHashSink(htSHA256);
+    HashSink fileHashSink { htSHA256 };
     std::shared_ptr<FSAccessor> narAccessor;
+    HashSink narHashSink { htSHA256 };
     {
     FdSink fileSink(fdTemp.get());
-    TeeSink teeSink(fileSink, fileHashSink);
-    auto compressionSink = makeCompressionSink(compression, teeSink);
-    TeeSource teeSource(narSource, *compressionSink);
+    TeeSink teeSinkCompressed { fileSink, fileHashSink };
+    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
+    TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
+    TeeSource teeSource { narSource, teeSinkUncompressed };
     narAccessor = makeNarAccessor(teeSource);
     compressionSink->finish();
     fileSink.flush();
@@ -176,9 +171,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
     auto now2 = std::chrono::steady_clock::now();
 
+    auto info = mkInfo(narHashSink.finish());
     auto narInfo = make_ref<NarInfo>(info);
-    narInfo->narSize = info.narSize;
-    narInfo->narHash = info.narHash;
+    narInfo->narSize = info.narSize; // FIXME needed?
+    narInfo->narHash = info.narHash; // FIXME needed?
     narInfo->compression = compression;
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;
@@ -300,6 +296,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
+
+    return narInfo->path;
+}
+
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    if (!repair && isValidPath(info.path)) {
+        // FIXME: copyNAR -> null sink
+        narSource.drain();
+        return;
+    }
+
+    (void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
+        /* FIXME reinstate these, once we can correctly do hash modulo sink as
+           needed. */
+        // assert(info.narHash == nar.first);
+        // assert(info.narSize == nar.second);
+        return info;
+    }});
+}
+
+StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
+{
+    if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
+        unsupported("addToStoreFromDump");
+    return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
+        ValidPathInfo info {
+            makeFixedOutputPath(method, nar.first, name),
+            nar.first,
+        };
+        info.narSize = nar.second;
+        return info;
+    });
 }
 
 bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
@@ -367,11 +398,9 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
-    // FIXME: some cut&paste from LocalStore::addToStore().
-
-    /* Read the whole path into memory. This is not a very scalable
-       method for very large paths, but `copyPath' is mainly used for
-       small files. */
+    /* FIXME: Make BinaryCacheStore::addToStoreCommon support
+       non-recursive+sha256 so we can just use the default
+       implementation of this method in terms of addToStoreFromDump. */
     StringSink sink;
     std::optional<Hash> h;
     if (method == FileIngestionMethod::Recursive) {
@@ -383,34 +412,25 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info {
-        makeFixedOutputPath(method, *h, name),
-        Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
-    };
-
     auto source = StringSource { *sink.s };
-    addToStore(info, source, repair, CheckSigs);
-
-    return std::move(info.path);
+    return addToStoreFromDump(source, name, FileIngestionMethod::Recursive, htSHA256, repair);
 }
 
 StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
-    ValidPathInfo info {
-        computeStorePathForText(name, s, references),
-        Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
-    };
-    info.references = references;
+    auto path = computeStorePathForText(name, s, references);
 
-    if (repair || !isValidPath(info.path)) {
-        StringSink sink;
-        dumpString(s, sink);
-        auto source = StringSource { *sink.s };
-        addToStore(info, source, repair, CheckSigs);
-    }
+    if (!repair && isValidPath(path))
+        return path;
 
-    return std::move(info.path);
+    auto source = StringSource { s };
+    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
+        ValidPathInfo info { path, nar.first };
+        info.narSize = nar.second;
+        info.references = references;
+        return info;
+    });
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 4b779cdd4..ce69ad3b4 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -72,6 +72,10 @@ private:
 
     void writeNarInfo(ref<NarInfo> narInfo);
 
+    StorePath addToStoreCommon(
+        Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
+        std::function<ValidPathInfo(HashResult)> mkInfo);
+
 public:
 
     bool isValidPathUncached(const StorePath & path) override;
@@ -85,6 +89,9 @@ public:
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
+
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 591140874..85a84d080 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -454,9 +454,7 @@ public:
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
-    {
-        throw Error("addToStoreFromDump() is not supported by this store");
-    }
+    { unsupported("addToStoreFromDump"); }
 
     /* Like addToStore, but the contents written to the output path is
        a regular file containing the given string. */

From 8a2e10827f2f57c3b9a0829357363d5f09af65e2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 14:08:23 +0200
Subject: [PATCH 267/882] Remove unused Flake::vOutputs field

---
 src/libexpr/flake/flake.cc | 5 ++---
 src/libexpr/flake/flake.hh | 1 -
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 783b98ef0..460eea5ea 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -215,10 +215,9 @@ static Flake getFlake(
 
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
         expectType(state, tLambda, *outputs->value, *outputs->pos);
-        flake.vOutputs = allocRootValue(outputs->value);
 
-        if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
-            for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
+        if (outputs->value->lambda.fun->matchAttrs) {
+            for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
                         .ref = parseFlakeRef(formal.name)
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index c2bb2888b..69c779af8 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -34,7 +34,6 @@ struct Flake
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
-    RootValue vOutputs;
     ~Flake();
 };
 

From 21639b2d179e84805d5ab0949c7bdd74c9e2b7ae Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 23 Sep 2020 16:05:47 +0200
Subject: [PATCH 268/882] Use gold as the linker on Linux

Saves ~7s in the linking phase
---
 flake.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flake.nix b/flake.nix
index 0304557e8..200417c3e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -58,6 +58,7 @@
         configureFlags =
           lib.optionals stdenv.isLinux [
             "--with-sandbox-shell=${sh}/bin/busybox"
+            "LDFLAGS=-fuse-ld=gold"
           ];
 
         buildDeps =

From 412b3a54fb02cdf49cb084a925bd14c24e14aea8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 23 Sep 2020 10:36:55 -0400
Subject: [PATCH 269/882] Clarify FIXME in `BinaryCacheStore::addToStoreCommon`

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libstore/binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 6679eb16f..817661869 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -311,7 +311,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
     (void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
         /* FIXME reinstate these, once we can correctly do hash modulo sink as
-           needed. */
+           needed. We need to throw here in case we uploaded a corrupted store path. */
         // assert(info.narHash == nar.first);
         // assert(info.narSize == nar.second);
         return info;

From 3f226f71c185b2fbaaabb01bd0f3ba3cd4a39612 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 23 Sep 2020 14:40:41 +0000
Subject: [PATCH 270/882] Return more info from
 `BinaryCacheStore::addToStoreCommon`

We don't need it yet, but we could/should in the future, and it's a
cost-free change since we already have the reference. I like it.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libstore/binary-cache-store.cc | 8 ++++----
 src/libstore/binary-cache-store.hh | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 817661869..f7a52a296 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -142,7 +142,7 @@ struct FileSource : FdSource
     }
 };
 
-StorePath BinaryCacheStore::addToStoreCommon(
+ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
     std::function<ValidPathInfo(HashResult)> mkInfo)
 {
@@ -297,7 +297,7 @@ StorePath BinaryCacheStore::addToStoreCommon(
 
     stats.narInfoWrite++;
 
-    return narInfo->path;
+    return narInfo;
 }
 
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
@@ -330,7 +330,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
         };
         info.narSize = nar.second;
         return info;
-    });
+    })->path;
 }
 
 bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
@@ -430,7 +430,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
         info.narSize = nar.second;
         info.references = references;
         return info;
-    });
+    })->path;
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index ce69ad3b4..5224d7ec8 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -72,7 +72,7 @@ private:
 
     void writeNarInfo(ref<NarInfo> narInfo);
 
-    StorePath addToStoreCommon(
+    ref<const ValidPathInfo> addToStoreCommon(
         Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
         std::function<ValidPathInfo(HashResult)> mkInfo);
 

From e8f0b1e996e95e4a1025976d6cfb7ece734129a2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 15:34:11 +0200
Subject: [PATCH 271/882] DerivationGoal::registerOutputs(): Fix bad format
 string

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 07c5bceb2..2e23dd979 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3858,7 +3858,7 @@ void DerivationGoal::registerOutputs()
            something like that. */
         canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
 
-        debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath);
+        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
 
         /* Pass blank Sink as we are not ready to hash data at this stage. */
         NullSink blank;

From d4f8163d104aee0f39e6d8b98160f8de12c18824 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 15:47:12 +0200
Subject: [PATCH 272/882] canonicalisePathMetaData_(): Change assertion to
 error message

---
 src/libstore/local-store.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c91f3fbf7..1f58977a5 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -478,8 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
        ensure that we don't fail on hard links within the same build
        (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
-        assert(!S_ISDIR(st.st_mode));
-        if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
+        if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
             throw BuildError("invalid ownership on file '%1%'", path);
         mode_t mode = st.st_mode & ~S_IFMT;
         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));

From cec94738715275ec4761071cefe4a9ae1a565960 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 15:47:30 +0200
Subject: [PATCH 273/882] DerivationGoal::registerOutputs(): Don't canonicalize
 twice

Fixes #4021.
---
 src/libstore/build.cc | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2e23dd979..2979ce010 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3913,7 +3913,6 @@ void DerivationGoal::registerOutputs()
                 outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
         };
 
-        bool rewritten = false;
         std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
             [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
                 finish(skippedFinalPath.path);
@@ -3943,8 +3942,6 @@ void DerivationGoal::registerOutputs()
                 sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
                 StringSource source(*sink.s);
                 restorePath(actualPath, source);
-
-                rewritten = true;
             }
         };
 
@@ -4125,11 +4122,6 @@ void DerivationGoal::registerOutputs()
             }
         }
 
-        /* Get rid of all weird permissions.  This also checks that
-           all files are owned by the build user, if applicable. */
-        canonicalisePathMetaData(actualPath,
-            buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen);
-
         if (buildMode == bmCheck) {
             if (!worker.store.isValidPath(newInfo.path)) continue;
             ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));

From 2548347bba2d4e6f9947dadf95ed24a16f8a1cdc Mon Sep 17 00:00:00 2001
From: Dominique Martinet <asmadeus@codewreck.org>
Date: Wed, 23 Sep 2020 17:39:19 +0200
Subject: [PATCH 274/882] libutil/archive: add preallocate-contents option

Make archive preallocation (fallocate) optional because some filesystems
like btrfs do not behave as expected with fallocate.

See #3550.
---
 src/libutil/archive.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 14399dea3..cdf3d89ae 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -27,6 +27,8 @@ struct ArchiveSettings : Config
         #endif
         "use-case-hack",
         "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
+    Setting<bool> preallocateContents{this, true, "preallocate-contents",
+        "Whether to preallocate files when writing objects with known size."};
 };
 
 static ArchiveSettings archiveSettings;
@@ -325,6 +327,9 @@ struct RestoreSink : ParseSink
 
     void preallocateContents(uint64_t len)
     {
+        if (!archiveSettings.preallocateContents)
+            return;
+
 #if HAVE_POSIX_FALLOCATE
         if (len) {
             errno = posix_fallocate(fd.get(), 0, len);

From 31ab4c3816675b5bf2f7b29dfb10112cd8ec9ceb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 19:09:58 +0200
Subject: [PATCH 275/882] Test whether build/repair results are read-only

---
 tests/repair.sh | 13 +++++--------
 tests/simple.sh |  4 +++-
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tests/repair.sh b/tests/repair.sh
index ec7ad5dca..ba019028d 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -13,14 +13,14 @@ hash=$(nix-hash $path2)
 chmod u+w $path2
 touch $path2/bad
 
-if nix-store --verify --check-contents -v; then
-    echo "nix-store --verify succeeded unexpectedly" >&2
-    exit 1
-fi
+(! nix-store --verify --check-contents -v)
 
 # The path can be repaired by rebuilding the derivation.
 nix-store --verify --check-contents --repair
 
+(! [ -e $path2/bad ])
+(! [ -w $path2 ])
+
 nix-store --verify-path $path2
 
 # Re-corrupt and delete the deriver. Now --verify --repair should
@@ -30,10 +30,7 @@ touch $path2/bad
 
 nix-store --delete $(nix-store -qd $path2)
 
-if nix-store --verify --check-contents --repair; then
-    echo "nix-store --verify --repair succeeded unexpectedly" >&2
-    exit 1
-fi
+(! nix-store --verify --check-contents --repair)
 
 nix-build dependencies.nix -o $TEST_ROOT/result --repair
 
diff --git a/tests/simple.sh b/tests/simple.sh
index 37631b648..15bd2bd16 100644
--- a/tests/simple.sh
+++ b/tests/simple.sh
@@ -10,13 +10,15 @@ outPath=$(nix-store -rvv "$drvPath")
 
 echo "output path is $outPath"
 
+(! [ -w $outPath ])
+
 text=$(cat "$outPath"/hello)
 if test "$text" != "Hello World!"; then exit 1; fi
 
 # Directed delete: $outPath is not reachable from a root, so it should
 # be deleteable.
 nix-store --delete $outPath
-if test -e $outPath/hello; then false; fi
+(! [ -e $outPath/hello ])
 
 outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)"
 if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then

From 688bd4fb500c70cda4ffe6864bbf59dbc4da49a2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 19:10:16 +0200
Subject: [PATCH 276/882] After rewriting a path, make it read-only

---
 src/libstore/build.cc | 47 ++++++++++++++-----------------------------
 1 file changed, 15 insertions(+), 32 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2979ce010..cf04fbe56 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3735,15 +3735,12 @@ void DerivationGoal::runChild()
 }
 
 
-static void moveCheckToStore(const Path & src, const Path & dst)
+/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
+   it's a directory and we're not root (to be able to update the
+   directory's parent link ".."). */
+static void movePath(const Path & src, const Path & dst)
 {
-    /* For the rename of directory to succeed, we must be running as root or
-       the directory must be made temporarily writable (to update the
-       directory's parent link ".."). */
-    struct stat st;
-    if (lstat(src.c_str(), &st) == -1) {
-        throw SysError("getting attributes of path '%1%'", src);
-    }
+    auto st = lstat(src);
 
     bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
 
@@ -3942,6 +3939,10 @@ void DerivationGoal::registerOutputs()
                 sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
                 StringSource source(*sink.s);
                 restorePath(actualPath, source);
+
+                /* FIXME: set proper permissions in restorePath() so
+                   we don't have to do another traversal. */
+                canonicalisePathMetaData(actualPath, -1, inodesSeen);
             }
         };
 
@@ -4024,7 +4025,7 @@ void DerivationGoal::registerOutputs()
             [&](DerivationOutputInputAddressed output) {
                 /* input-addressed case */
                 auto requiredFinalPath = output.path;
-                /* Preemtively add rewrite rule for final hash, as that is
+                /* Preemptively add rewrite rule for final hash, as that is
                    what the NAR hash will use rather than normalized-self references */
                 if (scratchPath != requiredFinalPath)
                     outputRewrites.insert_or_assign(
@@ -4098,27 +4099,9 @@ void DerivationGoal::registerOutputs()
                    else. No moving needed. */
                 assert(newInfo.ca);
             } else {
-                /* Temporarily add write perm so we can move, will be fixed
-                   later. */
-                {
-                    struct stat st;
-                    auto & mode = st.st_mode;
-                    if (lstat(actualPath.c_str(), &st))
-                        throw SysError("getting attributes of path '%1%'", actualPath);
-                    mode |= 0200;
-                    /* Try to change the perms, but only if the file isn't a
-                       symlink as symlinks permissions are mostly ignored and
-                       calling `chmod` on it will just forward the call to the
-                       target of the link. */
-                    if (!S_ISLNK(st.st_mode))
-                        if (chmod(actualPath.c_str(), mode) == -1)
-                            throw SysError("changing mode of '%1%' to %2$o", actualPath, mode);
-                }
-                if (rename(
-                        actualPath.c_str(),
-                        worker.store.toRealPath(finalDestPath).c_str()) == -1)
-                    throw SysError("moving build output '%1%' from it's temporary location to the Nix store", finalDestPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
+                auto destPath = worker.store.toRealPath(finalDestPath);
+                movePath(actualPath, destPath);
+                actualPath = destPath;
             }
         }
 
@@ -4128,9 +4111,9 @@ void DerivationGoal::registerOutputs()
             if (newInfo.narHash != oldInfo.narHash) {
                 worker.checkMismatch = true;
                 if (settings.runDiffHook || settings.keepFailed) {
-                    Path dst = worker.store.toRealPath(finalDestPath + checkSuffix);
+                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
                     deletePath(dst);
-                    moveCheckToStore(actualPath, dst);
+                    movePath(actualPath, dst);
 
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),

From 236d9ee7f72ca4238f5f44c244fd2b885691c6ad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 19:17:28 +0200
Subject: [PATCH 277/882] lstat() cleanup

---
 src/libexpr/parser.y                           |  3 +--
 src/libstore/build.cc                          |  9 ++-------
 src/libstore/gc.cc                             |  4 +---
 src/libstore/local-store.cc                    | 18 +++++-------------
 src/libstore/optimise-store.cc                 | 12 +++---------
 src/libstore/profiles.cc                       |  5 +----
 src/libutil/archive.cc                         |  4 +---
 .../resolve-system-dependencies.cc             |  6 +-----
 8 files changed, 15 insertions(+), 46 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 24b21f7da..a4c84c526 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -614,8 +614,7 @@ Path resolveExprPath(Path path)
         // Basic cycle/depth limit to avoid infinite loops.
         if (++followCount >= maxFollow)
             throw Error("too many symbolic links encountered while traversing the path '%s'", path);
-        if (lstat(path.c_str(), &st))
-            throw SysError("getting status of '%s'", path);
+        st = lstat(path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cf04fbe56..6b53f529a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2367,10 +2367,7 @@ void DerivationGoal::startBuilder()
         for (auto & i : inputPaths) {
             auto p = worker.store.printStorePath(i);
             Path r = worker.store.toRealPath(p);
-            struct stat st;
-            if (lstat(r.c_str(), &st))
-                throw SysError("getting attributes of path '%s'", p);
-            if (S_ISDIR(st.st_mode))
+            if (S_ISDIR(lstat(r).st_mode))
                 dirsInChroot.insert_or_assign(p, r);
             else
                 linkOrCopy(r, chrootRootDir + p);
@@ -3144,9 +3141,7 @@ void DerivationGoal::addDependency(const StorePath & path)
             if (pathExists(target))
                 throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
 
-            struct stat st;
-            if (lstat(source.c_str(), &st))
-                throw SysError("getting attributes of path '%s'", source);
+            auto st = lstat(source);
 
             if (S_ISDIR(st.st_mode)) {
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 08b53c702..518a357ef 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -663,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         if (name == "." || name == "..") continue;
         Path path = linksDir + "/" + name;
 
-        struct stat st;
-        if (lstat(path.c_str(), &st) == -1)
-            throw SysError("statting '%1%'", path);
+        auto st = lstat(path);
 
         if (st.st_nlink != 1) {
             actualSize += st.st_size;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1f58977a5..ee997ef3a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -114,8 +114,7 @@ LocalStore::LocalStore(const Params & params)
         Path path = realStoreDir;
         struct stat st;
         while (path != "/") {
-            if (lstat(path.c_str(), &st))
-                throw SysError("getting status of '%1%'", path);
+            st = lstat(path);
             if (S_ISLNK(st.st_mode))
                 throw Error(
                         "the path '%1%' is a symlink; "
@@ -419,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
 
 void canonicaliseTimestampAndPermissions(const Path & path)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
-    canonicaliseTimestampAndPermissions(path, st);
+    canonicaliseTimestampAndPermissions(path, lstat(path));
 }
 
 
@@ -440,9 +436,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     }
 #endif
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
     /* Really make sure that the path is of a supported type. */
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
@@ -521,9 +515,7 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
 
     /* On platforms that don't have lchown(), the top-level path can't
        be a symlink, since we can't change its ownership. */
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
@@ -1454,7 +1446,7 @@ static void makeMutable(const Path & path)
 {
     checkInterrupt();
 
-    struct stat st = lstat(path);
+    auto st = lstat(path);
 
     if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
 
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index e4b4b6213..c032a5e22 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -17,9 +17,7 @@ namespace nix {
 
 static void makeWritable(const Path & path)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
         throw SysError("changing writability of '%1%'", path);
 }
@@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
 {
     checkInterrupt();
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
 #if __APPLE__
     /* HFS/macOS has some undocumented security feature disabling hardlinking for
@@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
 
     /* Yes!  We've seen a file with the same contents.  Replace the
        current file with a hard link to that file. */
-    struct stat stLink;
-    if (lstat(linkPath.c_str(), &stLink))
-        throw SysError("getting attributes of path '%1%'", linkPath);
+    auto stLink = lstat(linkPath);
 
     if (st.st_ino == stLink.st_ino) {
         debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index c20386e2b..c3809bad7 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -39,13 +39,10 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
     for (auto & i : readDirectory(profileDir)) {
         if (auto n = parseName(profileName, i.name)) {
             auto path = profileDir + "/" + i.name;
-            struct stat st;
-            if (lstat(path.c_str(), &st) != 0)
-                throw SysError("statting '%1%'", path);
             gens.push_back({
                 .number = *n,
                 .path = path,
-                .creationTime = st.st_mtime
+                .creationTime = lstat(path).st_mtime
             });
         }
     }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 14399dea3..4f59c8ed5 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -66,9 +66,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
 {
     checkInterrupt();
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
     sink << "(";
 
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index 434ad80a6..d30227e4e 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -111,11 +111,7 @@ std::set<std::string> runResolver(const Path & filename)
 
 bool isSymlink(const Path & path)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st) == -1)
-        throw SysError("getting attributes of path '%1%'", path);
-
-    return S_ISLNK(st.st_mode);
+    return S_ISLNK(lstat(path).st_mode);
 }
 
 Path resolveSymlink(const Path & path)

From 9a24ece122eb19f3b69f072f6ce3c39c5ae4d0ce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 20:21:08 +0200
Subject: [PATCH 278/882] Fix exception

---
 src/libstore/build.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6b53f529a..f0820e711 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1675,7 +1675,7 @@ void DerivationGoal::tryLocalBuild() {
 }
 
 
-void replaceValidPath(const Path & storePath, const Path tmpPath)
+void replaceValidPath(const Path & storePath, const Path & tmpPath)
 {
     /* We can't atomically replace storePath (the original) with
        tmpPath (the replacement), so we have to move it out of the
@@ -1685,8 +1685,9 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
     if (pathExists(storePath))
         rename(storePath.c_str(), oldPath.c_str());
     if (rename(tmpPath.c_str(), storePath.c_str()) == -1) {
+        auto ex = SysError("moving '%s' to '%s'", tmpPath, storePath);
         rename(oldPath.c_str(), storePath.c_str()); // attempt to recover
-        throw SysError("moving '%s' to '%s'", tmpPath, storePath);
+        throw ex;
     }
     deletePath(oldPath);
 }

From 4ce8a3ed452f06b18a40cffefc37d47c916927a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Sep 2020 21:29:10 +0200
Subject: [PATCH 279/882] Hopefully fix EPERM on macOS

---
 src/libstore/build.cc | 72 ++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 32 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f0820e711..db7dbc17e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1675,6 +1675,33 @@ void DerivationGoal::tryLocalBuild() {
 }
 
 
+static void chmod_(const Path & path, mode_t mode)
+{
+    if (chmod(path.c_str(), mode) == -1)
+        throw SysError("setting permissions on '%s'", path);
+}
+
+
+/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
+   it's a directory and we're not root (to be able to update the
+   directory's parent link ".."). */
+static void movePath(const Path & src, const Path & dst)
+{
+    auto st = lstat(src);
+
+    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+    if (changePerm)
+        chmod_(src, st.st_mode | S_IWUSR);
+
+    if (rename(src.c_str(), dst.c_str()))
+        throw SysError("renaming '%1%' to '%2%'", src, dst);
+
+    if (changePerm)
+        chmod_(dst, st.st_mode);
+}
+
+
 void replaceValidPath(const Path & storePath, const Path & tmpPath)
 {
     /* We can't atomically replace storePath (the original) with
@@ -1683,12 +1710,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
        we're repairing (say) Glibc, we end up with a broken system. */
     Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
     if (pathExists(storePath))
-        rename(storePath.c_str(), oldPath.c_str());
-    if (rename(tmpPath.c_str(), storePath.c_str()) == -1) {
-        auto ex = SysError("moving '%s' to '%s'", tmpPath, storePath);
-        rename(oldPath.c_str(), storePath.c_str()); // attempt to recover
-        throw ex;
+        movePath(storePath, oldPath);
+
+    try {
+        movePath(tmpPath, storePath);
+    } catch (...) {
+        try {
+            // attempt to recover
+            movePath(oldPath, storePath);
+        } catch (...) {
+            ignoreException();
+        }
+        throw;
     }
+
     deletePath(oldPath);
 }
 
@@ -2006,13 +2041,6 @@ HookReply DerivationGoal::tryBuildHook()
 }
 
 
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
 int childEntry(void * arg)
 {
     ((DerivationGoal *) arg)->runChild();
@@ -3731,26 +3759,6 @@ void DerivationGoal::runChild()
 }
 
 
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output

From bd5f3dbe118d569ffb201ce14394572ac5fc412c Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Thu, 24 Sep 2020 12:30:03 -0700
Subject: [PATCH 280/882] Fixes fall-through to report correct description of
 hash-file command.

---
 src/nix/hash.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 0eca4f8ea..494f00a20 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -44,6 +44,7 @@ struct CmdHash : Command
         switch (mode) {
         case FileIngestionMethod::Flat:
             d = "print cryptographic hash of a regular file";
+            break;
         case FileIngestionMethod::Recursive:
             d = "print cryptographic hash of the NAR serialisation of a path";
         };

From ed218e1d6cf755fc3011c0954eb7031f95d3d732 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Fri, 25 Sep 2020 00:07:42 +0300
Subject: [PATCH 281/882] Fix max-jobs option

After 0ed946aa616bbf7ffe7f90d3309abdd27d875b10, max-jobs setting (-j/--max-jobs)
stopped working.

The reason was that nrLocalBuilds (which compared to maxBuildJobs to figure
out whether the limit is reached or not) is not incremented yet when tryBuild
is started; So, the solution is to move the check to tryLocalBuild.

Closes https://github.com/nixos/nix/issues/3763
---
 src/libstore/build.cc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index db7dbc17e..3fc24b221 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1612,6 +1612,13 @@ void DerivationGoal::tryToBuild()
 
     actLock.reset();
 
+    state = &DerivationGoal::tryLocalBuild;
+    worker.wakeUp(shared_from_this());
+}
+
+void DerivationGoal::tryLocalBuild() {
+    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+
     /* Make sure that we are allowed to start a build.  If this
        derivation prefers to be done locally, do it even if
        maxBuildJobs is 0. */
@@ -1622,12 +1629,6 @@ void DerivationGoal::tryToBuild()
         return;
     }
 
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
     if (settings.buildUsersGroup != "" && getuid() == 0) {

From 4d863a9fcb9460a9e4978466e03d2982d32e39f0 Mon Sep 17 00:00:00 2001
From: Paul Opiyo <59094545+paulopiyo777@users.noreply.github.com>
Date: Thu, 24 Sep 2020 18:05:47 -0500
Subject: [PATCH 282/882] Remove redundant value checks

std::optional had redundant checks for whether it had a value.
An object is emplaced either way so it can be dereferenced
without repeating a value check
---
 src/libexpr/flake/flake.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 460eea5ea..760ed1a6e 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -48,17 +48,17 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
                 resolvedRef = originalRef.resolve(state.store);
                 auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
                 if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
-                flakeCache.push_back({resolvedRef, fetchedResolved.value()});
-                fetched.emplace(fetchedResolved.value());
+                flakeCache.push_back({resolvedRef, *fetchedResolved});
+                fetched.emplace(*fetchedResolved);
             }
             else {
                 throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
             }
         }
-        flakeCache.push_back({originalRef, fetched.value()});
+        flakeCache.push_back({originalRef, *fetched});
     }
 
-    auto [tree, lockedRef] = fetched.value();
+    auto [tree, lockedRef] = *fetched;
 
     debug("got tree '%s' from '%s'",
         state.store->printStorePath(tree.storePath), lockedRef);

From 83fec38fc93922192ada7c0409fec76578ef8dfb Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Thu, 24 Sep 2020 22:41:24 -0700
Subject: [PATCH 283/882] Update document generation for empty json object
 values.

---
 doc/manual/generate-options.nix | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix
index 7afe279c3..3c31a4eec 100644
--- a/doc/manual/generate-options.nix
+++ b/doc/manual/generate-options.nix
@@ -13,7 +13,12 @@ concatStrings (map
       then "*empty*"
       else if isBool option.value
       then (if option.value then "`true`" else "`false`")
-      else "`" + toString option.value + "`") + "\n\n"
+      else
+        # n.b. a StringMap value type is specified as a string, but
+        # this shows the value type.  The empty stringmap is "null" in
+        # JSON, but that converts to "{ }" here.
+        (if isAttrs option.value then "`\"\"`"
+         else "`" + toString option.value + "`")) + "\n\n"
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

From a439e9488df6c13d0e44dd4816df98487d69f4c6 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Thu, 24 Sep 2020 22:42:59 -0700
Subject: [PATCH 284/882] Support StringMap configuration settings.

Allows Configuration values that are space-separated key=value pairs.
---
 src/libutil/config.cc | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 309d23b40..b14feb10d 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -268,6 +268,26 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
+template<> void BaseSetting<StringMap>::set(const std::string & str)
+{
+    auto kvpairs = tokenizeString<Strings>(str);
+    for (auto & s : kvpairs)
+    {
+        auto eq = s.find_first_of('=');
+        if (std::string::npos != eq)
+            value.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
+        // else ignored
+    }
+}
+
+template<> std::string BaseSetting<StringMap>::to_string() const
+{
+    Strings kvstrs;
+    std::transform(value.begin(), value.end(), back_inserter(kvstrs),
+        [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; });
+    return concatStringsSep(" ", kvstrs);
+}
+
 template class BaseSetting<int>;
 template class BaseSetting<unsigned int>;
 template class BaseSetting<long>;
@@ -278,6 +298,7 @@ template class BaseSetting<bool>;
 template class BaseSetting<std::string>;
 template class BaseSetting<Strings>;
 template class BaseSetting<StringSet>;
+template class BaseSetting<StringMap>;
 
 void PathSetting::set(const std::string & str)
 {

From c2f48cfcee501dd15690245d481d154444456f66 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Thu, 24 Sep 2020 22:46:03 -0700
Subject: [PATCH 285/882] Complete conversion of "url" to "host" with
 associated variable renaming.

Completes the change begun in commit 56f1e0d to consistently use the
"host" attribute for "github" and "gitlab" inputs instead of a "url"
attribute.
---
 src/libfetchers/github.cc | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index d8d0351b9..eb758bc5f 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -65,9 +65,9 @@ struct GitArchiveInputScheme : InputScheme
                     throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
                 ref = value;
             }
-            else if (name == "url") {
+            else if (name == "host") {
                 if (!std::regex_match(value, urlRegex))
-                    throw BadURL("URL '%s' contains an invalid instance url", url.url);
+                    throw BadURL("URL '%s' contains an invalid instance host", url.url);
                 host_url = value;
             }
             // FIXME: barf on unsupported attributes
@@ -82,7 +82,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("repo", path[1]);
         if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
         if (ref) input.attrs.insert_or_assign("ref", *ref);
-        if (host_url) input.attrs.insert_or_assign("url", *host_url);
+        if (host_url) input.attrs.insert_or_assign("host", *host_url);
 
         return input;
     }
@@ -92,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != type()) return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
                 throw Error("unsupported input attribute '%s'", name);
 
         getStrAttr(attrs, "owner");
@@ -208,9 +208,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
         Headers headers;
         std::string accessToken = settings.githubAccessToken.get();
@@ -230,9 +230,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
         std::string accessToken = settings.githubAccessToken.get();
@@ -246,9 +246,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
     void clone(const Input & input, const Path & destDir) override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
-                host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
             .clone(destDir);
     }
@@ -284,10 +284,14 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
     DownloadUrl getDownloadUrl(const Input & input) const override
     {
-        // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        // This endpoint has a rate limit threshold that may be
+        // server-specific and vary based whether the user is
+        // authenticated via an accessToken or not, but the usual rate
+        // is 10 reqs/sec/ip-addr.  See
+        // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
         std::string accessToken = settings.gitlabAccessToken.get();
@@ -302,10 +306,10 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
     void clone(const Input & input, const Path & destDir) override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
         Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
-                host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
             .clone(destDir);
     }

From 8fba2a8b54283ea1cf56ae75faf4ced5f3e8e4a1 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Thu, 24 Sep 2020 22:49:44 -0700
Subject: [PATCH 286/882] Update to use access-tokens configuration for
 github/gitlab access.

This change provides support for using access tokens with other
instances of GitHub and GitLab beyond just github.com and
gitlab.com (especially company-specific or foundation-specific
instances).

This change also provides the ability to specify the type of access
token being used, where different types may have different handling,
based on the forge type.
---
 src/libfetchers/github.cc | 100 ++++++++++++++++++++++----------------
 src/libstore/globals.hh   |  50 ++++++++++++++++++-
 2 files changed, 107 insertions(+), 43 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index eb758bc5f..0e0655367 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -5,6 +5,7 @@
 #include "store-api.hh"
 #include "types.hh"
 
+#include <optional>
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
@@ -12,13 +13,10 @@ namespace nix::fetchers {
 struct DownloadUrl
 {
     std::string url;
-    std::optional<std::pair<std::string, std::string>> access_token_header;
+    Headers headers;
 
-    DownloadUrl(const std::string & url)
-        : url(url) { }
-
-    DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
-        : url(url), access_token_header(access_token_header) { }
+    DownloadUrl(const std::string & url, const Headers & headers)
+        : url(url), headers(headers) { }
 };
 
 // A github or gitlab url
@@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::string type() = 0;
 
-    virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
+    virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
@@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
+    std::optional<std::string> getAccessToken(const std::string &host) const {
+        auto tokens = settings.accessTokens.get();
+        auto pat = tokens.find(host);
+        if (pat == tokens.end())
+            return std::nullopt;
+        return pat->second;
+    }
+
+    Headers makeHeadersWithAuthTokens(const std::string & host) const {
+        Headers headers;
+        auto accessToken = getAccessToken(host);
+        if (accessToken) {
+            auto hdr = accessHeaderFromToken(*accessToken);
+            if (hdr)
+                headers.push_back(*hdr);
+            else
+                warn("Unrecognized access token for host '%s'", host);
+        }
+        return headers;
+    }
+
     virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
 
     virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
@@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        Headers headers;
-        if (url.access_token_header) {
-            headers.push_back(*url.access_token_header);
-        }
-
-        auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
+        auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true);
 
         input.attrs.insert_or_assign("lastModified", lastModified);
 
@@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+        // Github supports PAT/OAuth2 tokens and HTTP Basic
+        // Authentication.  The former simply specifies the token, the
+        // latter can use the token as the password.  Only the first
+        // is used here. See
+        // https://developer.github.com/v3/#authentication and
+        // https://docs.github.com/en/developers/apps/authorizing-oath-apps
         return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
     }
 
@@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
+        Headers headers = makeHeadersWithAuthTokens(host);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "") {
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
-        } else {
-            return DownloadUrl(url);
-        }
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl(url, headers);
     }
 
     void clone(const Input & input, const Path & destDir) override
@@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
-        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+        // Gitlab supports 4 kinds of authorization, two of which are
+        // relevant here: OAuth2 and PAT (Private Access Token).  The
+        // user can indicate which token is used by specifying the
+        // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+        // If the <TYPE> is unrecognized, this will fall back to
+        // treating this simply has <HDRNAME>:<HDRVAL>.  See
+        // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+        auto fldsplit = token.find_first_of(':');
+        // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+        if ("OAuth2" == token.substr(0, fldsplit))
+            return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+        if ("PAT" == token.substr(0, fldsplit))
+            return std::make_pair("Private-token", token.substr(fldsplit+1));
+        warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
+        return std::nullopt;
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        // See rate limiting note below
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
-        std::string accessToken = settings.gitlabAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
+        Headers headers = makeHeadersWithAuthTokens(host);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.gitlabAccessToken.get();
-        if (accessToken != "") {
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
-        } else {
-            return DownloadUrl(url);
-        }
-
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl(url, headers);
     }
 
     void clone(const Input & input, const Path & destDir) override
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index b2e7610ee..646422399 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -863,8 +863,54 @@ public:
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
         "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
 
-    Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token",
-        "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."};
+    Setting<StringMap> accessTokens{this, {}, "access-tokens",
+        R"(
+          Access tokens used to access protected GitHub, GitLab, or
+          other locations requiring token-based authentication.
+
+          Access tokens are specified as a string made up of
+          space-separated `host=token` values.  The specific token
+          used is selected by matching the `host` portion against the
+          "host" specification of the input. The actual use of the
+          `token` value is determined by the type of resource being
+          accessed:
+
+          * Github: the token value is the OAUTH-TOKEN string obtained
+            as the Personal Access Token from the Github server (see
+            https://docs.github.com/en/developers/apps/authorizing-oath-apps).
+
+          * Gitlab: the token value is either the OAuth2 token or the
+            Personal Access Token (these are different types tokens
+            for gitlab, see
+            https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
+            The `token` value should be `type:tokenstring` where
+            `type` is either `OAuth2` or `PAT` to indicate which type
+            of token is being specified.
+
+          Example `~/.config/nix/nix.conf`:
+
+          ```
+          personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+          ```
+
+          Example `~/code/flake.nix`:
+
+          ```nix
+          input.foo = {
+            type="gitlab";
+            host="gitlab.mycompany.com";
+            owner="mycompany";
+            repo="pro";
+          };
+          ```
+
+          This example specifies three tokens, one each for accessing
+          github.com, gitlab.mycompany.com, and sourceforge.net.
+
+          The `input.foo` uses the "gitlab" fetcher, which might
+          requires specifying the token type along with the token
+          value.
+          )"};
 
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};

From 7b2ae472ff05a39cd635ac10dbbce3cd17b60c93 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Sep 2020 10:27:40 +0200
Subject: [PATCH 287/882] expectArg(): Respect the 'optional' flag

---
 src/libutil/args.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 3c1f87f7e..f41242e17 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -192,7 +192,7 @@ public:
     {
         expectArgs({
             .label = label,
-            .optional = true,
+            .optional = optional,
             .handler = {dest}
         });
     }

From ef2a14be190f7162e85e9bdd44dd45bd9ddfe391 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Fri, 25 Sep 2020 08:08:27 -0700
Subject: [PATCH 288/882] Fix reference to older name for access-tokens config
 value.

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 646422399..959ebe360 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -890,7 +890,7 @@ public:
           Example `~/.config/nix/nix.conf`:
 
           ```
-          personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+          access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
           ```
 
           Example `~/code/flake.nix`:

From 5a35cc29bffc88b88f883dfcdd1bb251eab53ecd Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Fri, 25 Sep 2020 08:09:56 -0700
Subject: [PATCH 289/882] Re-add support for github-access-token, but mark as
 deprecated.

---
 src/libfetchers/github.cc | 10 ++++++++++
 src/libstore/globals.hh   |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 0e0655367..443644639 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -146,7 +146,17 @@ struct GitArchiveInputScheme : InputScheme
         auto tokens = settings.accessTokens.get();
         auto pat = tokens.find(host);
         if (pat == tokens.end())
+        {
+            if ("github.com" == host)
+            {
+                auto oldcfg = settings.githubAccessToken.get();
+                if (!oldcfg.empty()) {
+                    warn("using deprecated 'github-access-token' config value; please use 'access-tokens' instead");
+                    return oldcfg;
+                }
+            }
             return std::nullopt;
+        }
         return pat->second;
     }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 959ebe360..bd36ffc17 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -861,7 +861,7 @@ public:
         )"};
 
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
-        "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
+        "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes (deprecated, please use 'access-tokens' instead)."};
 
     Setting<StringMap> accessTokens{this, {}, "access-tokens",
         R"(

From cfe791a638a3fdf53a2608f885c407bafc238094 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 25 Sep 2020 11:30:04 -0400
Subject: [PATCH 290/882] stdout_ -> cout

Better to get creative than just sprinkle arbitrary underscores.
---
 src/libutil/logging.hh  |  2 +-
 src/nix/add-to-store.cc |  2 +-
 src/nix/eval.cc         |  2 +-
 src/nix/flake.cc        | 36 ++++++++++++++++++------------------
 src/nix/hash.cc         |  4 ++--
 src/nix/ls.cc           |  4 ++--
 src/nix/profile.cc      |  2 +-
 src/nix/registry.cc     |  2 +-
 src/nix/search.cc       |  6 +++---
 src/nix/show-config.cc  |  4 ++--
 src/nix/why-depends.cc  |  2 +-
 11 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 77b92fb51..e3fe613e8 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -100,7 +100,7 @@ public:
     virtual void writeToStdout(std::string_view s);
 
     template<typename... Args>
-    inline void stdout_(const std::string & fs, const Args & ... args)
+    inline void cout(const std::string & fs, const Args & ... args)
     {
         boost::format f(fs);
         formatHelper(f, args...);
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index df55d1bc4..04ab664b3 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -83,7 +83,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             store->addToStore(info, source);
         }
 
-        logger->stdout_("%s", store->printStorePath(info.path));
+        logger->cout("%s", store->printStorePath(info.path));
     }
 };
 
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 9689f2bdb..754ffc911 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableCommand
             printValueAsJSON(*state, true, *v, jsonOut, context);
         } else {
             state->forceValueDeep(*v);
-            logger->stdout_("%s", *v);
+            logger->cout("%s", *v);
         }
     }
 };
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 64fc896d9..90ffaad7c 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -62,17 +62,17 @@ public:
 
 static void printFlakeInfo(const Store & store, const Flake & flake)
 {
-    logger->stdout_("Resolved URL:  %s", flake.resolvedRef.to_string());
-    logger->stdout_("Locked URL:    %s", flake.lockedRef.to_string());
+    logger->cout("Resolved URL:  %s", flake.resolvedRef.to_string());
+    logger->cout("Locked URL:    %s", flake.lockedRef.to_string());
     if (flake.description)
-        logger->stdout_("Description:   %s", *flake.description);
-    logger->stdout_("Path:          %s", store.printStorePath(flake.sourceInfo->storePath));
+        logger->cout("Description:   %s", *flake.description);
+    logger->cout("Path:          %s", store.printStorePath(flake.sourceInfo->storePath));
     if (auto rev = flake.lockedRef.input.getRev())
-        logger->stdout_("Revision:      %s", rev->to_string(Base16, false));
+        logger->cout("Revision:      %s", rev->to_string(Base16, false));
     if (auto revCount = flake.lockedRef.input.getRevCount())
-        logger->stdout_("Revisions:     %s", *revCount);
+        logger->cout("Revisions:     %s", *revCount);
     if (auto lastModified = flake.lockedRef.input.getLastModified())
-        logger->stdout_("Last modified: %s",
+        logger->cout("Last modified: %s",
             std::put_time(std::localtime(&*lastModified), "%F %T"));
 }
 
@@ -140,7 +140,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
 
         if (json) {
             auto json = flakeToJson(*store, flake);
-            logger->stdout_("%s", json.dump());
+            logger->cout("%s", json.dump());
         } else
             printFlakeInfo(*store, flake);
     }
@@ -158,9 +158,9 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
         auto flake = lockFlake();
 
         if (json)
-            logger->stdout_("%s", flake.lockFile.toJson());
+            logger->cout("%s", flake.lockFile.toJson());
         else {
-            logger->stdout_("%s", flake.flake.lockedRef);
+            logger->cout("%s", flake.flake.lockedRef);
 
             std::unordered_set<std::shared_ptr<Node>> visited;
 
@@ -172,7 +172,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
                     bool last = i + 1 == node.inputs.size();
 
                     if (auto lockedNode = std::get_if<0>(&input.second)) {
-                        logger->stdout_("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
+                        logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
                             prefix + (last ? treeLast : treeConn), input.first,
                             *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef);
 
@@ -180,7 +180,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
 
                         if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine));
                     } else if (auto follows = std::get_if<1>(&input.second)) {
-                        logger->stdout_("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'",
+                        logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'",
                             prefix + (last ? treeLast : treeConn), input.first,
                             printInputPath(*follows));
                     }
@@ -811,7 +811,7 @@ struct CmdFlakeShow : FlakeCommand
             try {
                 auto recurse = [&]()
                 {
-                    logger->stdout_("%s", headerPrefix);
+                    logger->cout("%s", headerPrefix);
                     auto attrs = visitor.getAttrs();
                     for (const auto & [i, attr] : enumerate(attrs)) {
                         bool last = i + 1 == attrs.size();
@@ -837,7 +837,7 @@ struct CmdFlakeShow : FlakeCommand
                     }
                     */
 
-                    logger->stdout_("%s: %s '%s'",
+                    logger->cout("%s: %s '%s'",
                         headerPrefix,
                         attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" :
                         attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" :
@@ -885,7 +885,7 @@ struct CmdFlakeShow : FlakeCommand
                     if (attrPath.size() == 1)
                         recurse();
                     else if (!showLegacy)
-                        logger->stdout_("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
+                        logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
                     else {
                         if (visitor.isDerivation())
                             showDerivation();
@@ -902,7 +902,7 @@ struct CmdFlakeShow : FlakeCommand
                     auto aType = visitor.maybeGetAttr("type");
                     if (!aType || aType->getString() != "app")
                         throw EvalError("not an app definition");
-                    logger->stdout_("%s: app", headerPrefix);
+                    logger->cout("%s: app", headerPrefix);
                 }
 
                 else if (
@@ -910,11 +910,11 @@ struct CmdFlakeShow : FlakeCommand
                     (attrPath.size() == 2 && attrPath[0] == "templates"))
                 {
                     auto description = visitor.getAttr("description")->getString();
-                    logger->stdout_("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
+                    logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
                 }
 
                 else {
-                    logger->stdout_("%s: %s",
+                    logger->cout("%s: %s",
                         headerPrefix,
                         attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" :
                         attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" :
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 52e41e50c..945d8e990 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -73,7 +73,7 @@ struct CmdHash : Command
 
             Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
-            logger->stdout_(h.to_string(base, base == SRI));
+            logger->cout(h.to_string(base, base == SRI));
         }
     }
 };
@@ -107,7 +107,7 @@ struct CmdToBase : Command
     void run() override
     {
         for (auto s : args)
-            logger->stdout_(Hash::parseAny(s, ht).to_string(base, base == SRI));
+            logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI));
     }
 };
 
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index b1cf92692..a9664e7c3 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -37,11 +37,11 @@ struct MixLs : virtual Args, MixJSON
                 auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
                 if (st.type == FSAccessor::Type::tSymlink)
                     line += " -> " + accessor->readLink(curPath);
-                logger->stdout_(line);
+                logger->cout(line);
                 if (recursive && st.type == FSAccessor::Type::tDirectory)
                     doPath(st, curPath, relPath, false);
             } else {
-                logger->stdout_(relPath);
+                logger->cout(relPath);
                 if (recursive) {
                     auto st = accessor->stat(curPath);
                     if (st.type == FSAccessor::Type::tDirectory)
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index f97df4d9e..def7db03b 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -389,7 +389,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
 
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
-            logger->stdout_("%d %s %s %s", i,
+            logger->cout("%d %s %s %s", i,
                 element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-",
                 element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-",
                 concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index afa3503b8..941785e55 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -26,7 +26,7 @@ struct CmdRegistryList : StoreCommand
         for (auto & registry : registries) {
             for (auto & entry : registry->entries) {
                 // FIXME: format nicely
-                logger->stdout_("%s %s %s",
+                logger->cout("%s %s %s",
                     registry->type == Registry::Flag   ? "flags " :
                     registry->type == Registry::User   ? "user  " :
                     registry->type == Registry::System ? "system" :
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 88815efdb..2f7eb23bb 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -147,13 +147,13 @@ struct CmdSearch : InstallableCommand, MixJSON
                             jsonElem.attr("description", description);
                         } else {
                             auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
-                            if (results > 1) logger->stdout_("");
-                            logger->stdout_(
+                            if (results > 1) logger->cout("");
+                            logger->cout(
                                 "* %s%s",
                                 wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
                                 name.version != "" ? " (" + name.version + ")" : "");
                             if (description != "")
-                                logger->stdout_(
+                                logger->cout(
                                     "  %s", hilite(description, descriptionMatch, ANSI_NORMAL));
                         }
                     }
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 01a49f107..328cd2ff2 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -20,12 +20,12 @@ struct CmdShowConfig : Command, MixJSON
     {
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
-            logger->stdout_("%s", globalConfig.toJSON().dump());
+            logger->cout("%s", globalConfig.toJSON().dump());
         } else {
             std::map<std::string, Config::SettingInfo> settings;
             globalConfig.getSettings(settings);
             for (auto & s : settings)
-                logger->stdout_("%s = %s", s.first, s.second.value);
+                logger->cout("%s = %s", s.first, s.second.value);
         }
     }
 };
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index cbfc9b948..f49d19ab2 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -156,7 +156,7 @@ struct CmdWhyDepends : SourceExprCommand
             auto pathS = store->printStorePath(node.path);
 
             assert(node.dist != inf);
-            logger->stdout_("%s%s%s%s" ANSI_NORMAL,
+            logger->cout("%s%s%s%s" ANSI_NORMAL,
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
                 firstPad != "" ? "→ " : "",

From cb186f1e7536c9448455bfbf8dec16ad6600e88e Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Fri, 25 Sep 2020 09:36:18 -0700
Subject: [PATCH 291/882] Use "?dir=..." portion of "registry add" local path
 specification.

The registry targets generally follow a URL formatting schema with
support for a query parameter of "?dir=subpath" to specify a sub-path
location below the URL root.

Alternatively, an absolute path can be specified.  This specification
mode accepts the query parameter but ignores/drops it.  It would
probably be better to either (a) disallow the query parameter for the
path form, or (b) recognize the query parameter and add to the path.

This patch implements (b) for consistency, and to make it easier for
tooling that might switch between a remote git reference and a local
path reference.

See also issue #4050.
---
 src/libexpr/flake/flakeref.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
index d5c2ffe66..762e27e1f 100644
--- a/src/libexpr/flake/flakeref.cc
+++ b/src/libexpr/flake/flakeref.cc
@@ -157,7 +157,8 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         } else {
             if (!hasPrefix(path, "/"))
                 throw BadURL("flake reference '%s' is not an absolute path", url);
-            path = canonPath(path);
+            auto query = decodeQuery(match[2]);
+            path = canonPath(path + "/" + get(query, "dir").value_or(""));
         }
 
         fetchers::Attrs attrs;

From 5db83dd771b92bcacb0cd4dea7d4e06f767769ca Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 26 Sep 2020 03:21:36 +0000
Subject: [PATCH 292/882] BinaryCacheStore::addTextToStore include CA field

---
 src/libstore/binary-cache-store.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f7a52a296..a3a73fe27 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -419,7 +419,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
 StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
-    auto path = computeStorePathForText(name, s, references);
+    auto textHash = hashString(htSHA256, s);
+    auto path = makeTextPath(name, textHash, references);
 
     if (!repair && isValidPath(path))
         return path;
@@ -428,6 +429,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info { path, nar.first };
         info.narSize = nar.second;
+        info.ca = TextHash { textHash };
         info.references = references;
         return info;
     })->path;

From 1832436526307ac92baec0146b89e9a5cf3aca35 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 26 Sep 2020 04:56:29 +0000
Subject: [PATCH 293/882] Fix up BinaryCacheStore::addToStore taking a path

---
 src/libstore/binary-cache-store.cc | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index a3a73fe27..b34b10fd1 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -401,19 +401,30 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
-    StringSink sink;
-    std::optional<Hash> h;
+
+    HashSink sink { hashAlgo };
     if (method == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
-        h = hashString(hashAlgo, *sink.s);
     } else {
-        auto s = readFile(srcPath);
-        dumpString(s, sink);
-        h = hashString(hashAlgo, s);
+        readFile(srcPath, sink);
     }
+    auto h = sink.finish().first;
 
-    auto source = StringSource { *sink.s };
-    return addToStoreFromDump(source, name, FileIngestionMethod::Recursive, htSHA256, repair);
+    auto source = sinkToSource([&](Sink & sink) {
+        dumpPath(srcPath, sink, filter);
+    });
+    return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
+        ValidPathInfo info {
+            makeFixedOutputPath(method, h, name),
+            nar.first,
+        };
+        info.narSize = nar.second;
+        info.ca = FixedOutputHash {
+            .method = method,
+            .hash = h,
+        };
+        return info;
+    })->path;
 }
 
 StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,

From 8b4a542d1767e0df7b3c0902b766f34352cb0958 Mon Sep 17 00:00:00 2001
From: Mateusz Piotrowski <0mp@FreeBSD.org>
Date: Sat, 26 Sep 2020 13:33:04 +0200
Subject: [PATCH 294/882] Fix a typo (#4073)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3cf4e44fa..11fe5f932 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ for more details.
 
 ## Installation
 
-On Linux and macOS the easiest way to Install Nix is to run the following shell command
+On Linux and macOS the easiest way to install Nix is to run the following shell command
 (as a user other than root):
 
 ```console

From 25fffdda865e51f68e72e0ca1775800b60391820 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 26 Sep 2020 10:17:30 -0400
Subject: [PATCH 295/882] Remove redundant nar hash and size setting

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libstore/binary-cache-store.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index b34b10fd1..59d02cab5 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -173,8 +173,6 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
 
     auto info = mkInfo(narHashSink.finish());
     auto narInfo = make_ref<NarInfo>(info);
-    narInfo->narSize = info.narSize; // FIXME needed?
-    narInfo->narHash = info.narHash; // FIXME needed?
     narInfo->compression = compression;
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;

From a76fb07314d3c5dea06ac2c1a36f8af1e76c2dde Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 26 Sep 2020 17:38:11 +0200
Subject: [PATCH 296/882] libmain/progress-bar: don't trim whitespace on the
 left

When running `nix build -L` it can be fairly hard to read the output if
the build program intentionally renders whitespace on the left. A
typical example is `g++` displaying compilation errors.

With this patch, the whitespace on the left is retained to make the log
more readable:

```
foo> no configure script, doing nothing
foo> building
foo> foobar.cc: In function 'int main()':
foo> foobar.cc:5:5: error: 'wrong_func' was not declared in this scope
foo>     5 |     wrong_func(1);
foo>       |     ^~~~~~~~~~
error: --- Error ------------------------------------------------------------------------------------- nix
error: --- Error --- nix-daemon
builder for '/nix/store/i1q76cw6cyh91raaqg5p5isd1l2x6rx2-foo-1.0.drv' failed with exit code 1
```
---
 src/libmain/progress-bar.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index be3c06a38..07b45b3b5 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -256,7 +256,7 @@ public:
         }
 
         else if (type == resBuildLogLine || type == resPostBuildLogLine) {
-            auto lastLine = trim(getS(fields, 0));
+            auto lastLine = chomp(getS(fields, 0));
             if (!lastLine.empty()) {
                 auto i = state->its.find(act);
                 assert(i != state->its.end());

From bd5328814fe8055b3f832a087afcf3ef11b06372 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Sat, 26 Sep 2020 14:32:58 -0700
Subject: [PATCH 297/882] Add some internal documentation for flake support
 objects.

---
 src/libexpr/flake/flake.hh    | 29 +++++++++++++++++++++++++----
 src/libexpr/flake/flakeref.hh | 15 +++++++++++++++
 src/libfetchers/fetchers.hh   | 16 ++++++++++++++++
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 69c779af8..40476a137 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -17,20 +17,41 @@ struct FlakeInput;
 
 typedef std::map<FlakeId, FlakeInput> FlakeInputs;
 
+// FlakeInput is the flake-level parsed form of the "input" entries in
+// the flake file.
+//
+// A FlakeInput is normally constructed by initially
+// first constructing a FlakeRef (a fetcher, the fetcher-specific
+// representation of the input specification, and the fetched local
+// store path result) and then creating this FlakeInput to hold that
+// FlakeRef, along with anything that might override that FlakeRef
+// (like command-line overrides or "follows" specifications).
+//
+// A FlakeInput is also sometimes constructed directly from a FlakeRef
+// instead of starting at the flake-file input specification
+// (e.g. overrides, follows, and implicit inputs).
+//
+// A FlakeInput will usually have one of either "ref" or "follows"
+// set.  If not otherwise specified, a "ref" will be generated to a
+// 'type="indirect"' flake, which is treated as simply the name of a
+// flake to be resolved in the registry.
+
 struct FlakeInput
 {
     std::optional<FlakeRef> ref;
-    bool isFlake = true;
+    bool isFlake = true;  // true = process flake to get outputs, false = (fetched) static source path
     std::optional<InputPath> follows;
     bool absolute = false; // whether 'follows' is relative to the flake root
     FlakeInputs overrides;
 };
 
+// The Flake structure is the main internal representation of a flake.nix file.
+
 struct Flake
 {
-    FlakeRef originalRef;
-    FlakeRef resolvedRef;
-    FlakeRef lockedRef;
+    FlakeRef originalRef;   // the original flake specification (by the user)
+    FlakeRef resolvedRef;   // registry references and caching resolved to the specific underlying flake
+    FlakeRef lockedRef;     // the specific local store result of invoking the fetcher
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh
index f4eb825a6..ac68cde0e 100644
--- a/src/libexpr/flake/flakeref.hh
+++ b/src/libexpr/flake/flakeref.hh
@@ -12,10 +12,25 @@ class Store;
 
 typedef std::string FlakeId;
 
+// The FlakeRef represents a local nix store reference to a flake
+// input for a Flake (it may be helpful to think of this object by the
+// alternate name of "InputRefForFlake").  It is constructed by
+// starting with an input description (usually the attrs or a url from
+// the flake file), locating a fetcher for that input, and then
+// capturing the Input object that fetcher generates (usually via
+// FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
+//
+// The actual fetch not have been performed yet (i.e. a FlakeRef may
+// be lazy), but the fetcher can be invoked at any time via the
+// FlakeRef to ensure the store is populated with this input.
+
 struct FlakeRef
 {
+    // fetcher-specific representation of the input, sufficient to
+    // perform the fetch operation.
     fetchers::Input input;
 
+    // sub-path within the fetched input that represents this input
     Path subdir;
 
     bool operator==(const FlakeRef & other) const;
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 89b1e6e7d..f361edf17 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -21,6 +21,13 @@ struct Tree
 
 struct InputScheme;
 
+// The Input object is generated by a specific fetcher, based on the
+// user-supplied input attribute in the flake.nix file, and contians
+// the information that the specific fetcher needs to perform the
+// actual fetch.  The Input object is most commonly created via the
+// "fromURL()" or "fromAttrs()" static functions which are provided the
+// url or attrset specified in the flake file.
+
 struct Input
 {
     friend struct InputScheme;
@@ -82,6 +89,15 @@ public:
     std::optional<time_t> getLastModified() const;
 };
 
+
+// The InputScheme represents a type of fetcher.  Each fetcher
+// registers with nix at startup time.  When processing an input for a
+// flake, each scheme is given an opportunity to "recognize" that
+// input from the url or attributes in the flake file's specification
+// and return an Input object to represent the input if it is
+// recognized.  The Input object contains the information the fetcher
+// needs to actually perform the "fetch()" when called.
+
 struct InputScheme
 {
     virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;

From 5885b0cfd878b4b60556c5b03bbe52244d04191a Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Sun, 27 Sep 2020 13:04:06 -0700
Subject: [PATCH 298/882] Miscellaneous spelling fixes in comments. (#4071)

---
 src/libexpr/flake/flake.cc  | 2 +-
 src/libfetchers/fetchers.hh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 760ed1a6e..b4ede542c 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -247,7 +247,7 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup
 }
 
 /* Compute an in-memory lock file for the specified top-level flake,
-   and optionally write it to file, it the flake is writable. */
+   and optionally write it to file, if the flake is writable. */
 LockedFlake lockFlake(
     EvalState & state,
     const FlakeRef & topRef,
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 89b1e6e7d..191e91978 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -73,7 +73,7 @@ public:
 
     StorePath computeStorePath(Store & store) const;
 
-    // Convience functions for common attributes.
+    // Convenience functions for common attributes.
     std::string getType() const;
     std::optional<Hash> getNarHash() const;
     std::optional<std::string> getRef() const;

From 3655875483306fa893ec2b01295151819a00ccaa Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 27 Sep 2020 22:35:03 +0200
Subject: [PATCH 299/882] doc/manual: update hacking docs (#4078)

* By default, build artifacts should be installed into `outputs/` rather
  than `inst/`[1].
* Add instructions on how to run unit-tests.

[1] 733d2e9402807e54d503c3113e854bfddb3d44e0
---
 doc/manual/src/hacking.md | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 5bd884ce8..2a1e55e5b 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -39,17 +39,17 @@ To build Nix itself in this shell:
 
 ```console
 [nix-shell]$ ./bootstrap.sh
-[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/inst
+[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out
 [nix-shell]$ make -j $NIX_BUILD_CORES
 ```
 
-To install it in `$(pwd)/inst` and test it:
+To install it in `$(pwd)/outputs` and test it:
 
 ```console
 [nix-shell]$ make install
-[nix-shell]$ make installcheck
-[nix-shell]$ ./inst/bin/nix --version
-nix (Nix) 2.4
+[nix-shell]$ make installcheck -j $NIX_BUILD_CORES
+[nix-shell]$ ./outputs/out/bin/nix --version
+nix (Nix) 3.0
 ```
 
 To run a functional test:
@@ -58,6 +58,12 @@ To run a functional test:
 make tests/test-name-should-auto-complete.sh.test
 ```
 
+To run the unit-tests for C++ code:
+
+```
+make check
+```
+
 If you have a flakes-enabled Nix you can replace:
 
 ```console

From 095a91f55a89d856e51ff099686e2bbe6fb9a384 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Sep 2020 05:37:07 +0000
Subject: [PATCH 300/882] Bump cachix/install-nix-action from v10 to v11

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v10 to v11.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v10...95a8068e317b8def9482980abe762f36c77ccc99)

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1f504a8ea..2f5c548d9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,7 @@ jobs:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v10
+    - uses: cachix/install-nix-action@v11
       with:
         skip_adding_nixpkgs_channel: true
     #- run: nix flake check

From ed66d010659a710646f5f1015c97a58614a713b3 Mon Sep 17 00:00:00 2001
From: Mateusz Piotrowski <0mp@FreeBSD.org>
Date: Mon, 28 Sep 2020 15:23:21 +0200
Subject: [PATCH 301/882] Fix tar invocation on FreeBSD

tar(1) on FreeBSD does not use standard output or input when the -f flag
is not provided. Instead, it defaults to /dev/sa0 on FreeBSD.

Make this tar invocation a bit more robust and explicitly tell tar(1) to
use standard output.

This is one of the issues discovered while porting Nix to FreeBSD. It has
been tested and committed locally to FreeBSD ports:
https://svnweb.freebsd.org/ports/head/sysutils/nix/Makefile?revision=550026&view=markup#l108
---
 tests/tarball.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/tarball.sh b/tests/tarball.sh
index 88a1a07a0..fe65a22e4 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -17,7 +17,7 @@ test_tarball() {
     local compressor="$2"
 
     tarball=$TEST_ROOT/tarball.tar$ext
-    (cd $TEST_ROOT && tar c tarball) | $compressor > $tarball
+    (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball
 
     nix-env -f file://$tarball -qa --out-path | grep -q dependencies
 

From 6c31297d80b71ab9c2c1084fae22f726f7d89daa Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 28 Sep 2020 11:32:58 -0400
Subject: [PATCH 302/882] Update src/libstore/binary-cache-store.cc

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 59d02cab5..f6224d6a0 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -307,7 +307,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
         return;
     }
 
-    (void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
+    addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
         /* FIXME reinstate these, once we can correctly do hash modulo sink as
            needed. We need to throw here in case we uploaded a corrupted store path. */
         // assert(info.narHash == nar.first);

From 80e335bb5837d7bfbf1f14d4f3d39525013c4f4d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 28 Sep 2020 15:43:56 +0000
Subject: [PATCH 303/882] Use `drvPath2` and give it a better name

---
 src/libstore/build.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0a6f2ccb5..0499273a4 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4315,19 +4315,19 @@ void DerivationGoal::registerOutputs()
        but it's fine to do in all cases. */
     bool isCaFloating = drv->type() == DerivationType::CAFloating;
 
-    auto drvPath2 = drvPath;
+    auto drvPathResolved = drvPath;
     if (!useDerivation && isCaFloating) {
         /* Once a floating CA derivations reaches this point, it
            must already be resolved, so we don't bother trying to
            downcast drv to get would would just be an empty
            inputDrvs field. */
         Derivation drv2 { *drv };
-        drvPath2 = writeDerivation(worker.store, drv2);
+        drvPathResolved = writeDerivation(worker.store, drv2);
     }
 
     if (useDerivation || isCaFloating)
         for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
+            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
 }
 
 

From bcb3da3b6b510bd24f2bc973b39bf43d92fad7ce Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Mon, 28 Sep 2020 08:58:14 -0700
Subject: [PATCH 304/882] Fix spelling error.

---
 src/libfetchers/fetchers.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index f361edf17..6aa8a3422 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -22,7 +22,7 @@ struct Tree
 struct InputScheme;
 
 // The Input object is generated by a specific fetcher, based on the
-// user-supplied input attribute in the flake.nix file, and contians
+// user-supplied input attribute in the flake.nix file, and contains
 // the information that the specific fetcher needs to perform the
 // actual fetch.  The Input object is most commonly created via the
 // "fromURL()" or "fromAttrs()" static functions which are provided the

From 5ae164b7cf2dd7ca1846f349b57131913aa7cf55 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Mon, 28 Sep 2020 09:23:05 -0700
Subject: [PATCH 305/882] Update description of FlakeRef, incorporating
 suggestion.

---
 src/libexpr/flake/flakeref.hh | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh
index ac68cde0e..2596dd18e 100644
--- a/src/libexpr/flake/flakeref.hh
+++ b/src/libexpr/flake/flakeref.hh
@@ -12,12 +12,19 @@ class Store;
 
 typedef std::string FlakeId;
 
-// The FlakeRef represents a local nix store reference to a flake
-// input for a Flake (it may be helpful to think of this object by the
-// alternate name of "InputRefForFlake").  It is constructed by
-// starting with an input description (usually the attrs or a url from
-// the flake file), locating a fetcher for that input, and then
-// capturing the Input object that fetcher generates (usually via
+// A flake reference specifies how to fetch a flake or raw source
+// (e.g. from a Git repository).  It is created from a URL-like syntax
+// (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
+// type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
+// path.
+//
+// Each flake will have a number of FlakeRef objects: one for each
+// input to the flake.
+//
+// The normal method of constructing a FlakeRef is by starting with an
+// input description (usually the attrs or a url from the flake file),
+// locating a fetcher for that input, and then capturing the Input
+// object that fetcher generates (usually via
 // FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
 //
 // The actual fetch not have been performed yet (i.e. a FlakeRef may

From 128c98ab0961ba234774508663f591758d3a2178 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Mon, 28 Sep 2020 09:34:23 -0700
Subject: [PATCH 306/882] Clarification in the description of the FlakeInput.

---
 src/libexpr/flake/flake.hh | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 40476a137..44cf636b5 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -17,15 +17,16 @@ struct FlakeInput;
 
 typedef std::map<FlakeId, FlakeInput> FlakeInputs;
 
-// FlakeInput is the flake-level parsed form of the "input" entries in
+// FlakeInput is the 'Flake'-level parsed form of the "input" entries in
 // the flake file.
 //
-// A FlakeInput is normally constructed by initially
-// first constructing a FlakeRef (a fetcher, the fetcher-specific
-// representation of the input specification, and the fetched local
-// store path result) and then creating this FlakeInput to hold that
-// FlakeRef, along with anything that might override that FlakeRef
-// (like command-line overrides or "follows" specifications).
+// A FlakeInput is normally constructed by the 'parseFlakeInput'
+// function which parses the input specification in the '.flake' file
+// to create a 'FlakeRef' (a fetcher, the fetcher-specific
+// representation of the input specification, and possibly the fetched
+// local store path result) and then creating this FlakeInput to hold
+// that FlakeRef, along with anything that might override that
+// FlakeRef (like command-line overrides or "follows" specifications).
 //
 // A FlakeInput is also sometimes constructed directly from a FlakeRef
 // instead of starting at the flake-file input specification

From 887be7b6f233822f03d156dafa156276d59162fd Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Mon, 28 Sep 2020 09:37:26 -0700
Subject: [PATCH 307/882] Switch comment format from '// ...' to '/* ... */'
 for consistency.

---
 src/libexpr/flake/flake.hh    | 39 +++++++++++++++----------------
 src/libexpr/flake/flakeref.hh | 43 ++++++++++++++++++-----------------
 src/libfetchers/fetchers.hh   | 28 ++++++++++++-----------
 3 files changed, 57 insertions(+), 53 deletions(-)

diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 44cf636b5..cf62c7741 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -17,25 +17,26 @@ struct FlakeInput;
 
 typedef std::map<FlakeId, FlakeInput> FlakeInputs;
 
-// FlakeInput is the 'Flake'-level parsed form of the "input" entries in
-// the flake file.
-//
-// A FlakeInput is normally constructed by the 'parseFlakeInput'
-// function which parses the input specification in the '.flake' file
-// to create a 'FlakeRef' (a fetcher, the fetcher-specific
-// representation of the input specification, and possibly the fetched
-// local store path result) and then creating this FlakeInput to hold
-// that FlakeRef, along with anything that might override that
-// FlakeRef (like command-line overrides or "follows" specifications).
-//
-// A FlakeInput is also sometimes constructed directly from a FlakeRef
-// instead of starting at the flake-file input specification
-// (e.g. overrides, follows, and implicit inputs).
-//
-// A FlakeInput will usually have one of either "ref" or "follows"
-// set.  If not otherwise specified, a "ref" will be generated to a
-// 'type="indirect"' flake, which is treated as simply the name of a
-// flake to be resolved in the registry.
+/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
+ * in the flake file.
+ *
+ * A FlakeInput is normally constructed by the 'parseFlakeInput'
+ * function which parses the input specification in the '.flake' file
+ * to create a 'FlakeRef' (a fetcher, the fetcher-specific
+ * representation of the input specification, and possibly the fetched
+ * local store path result) and then creating this FlakeInput to hold
+ * that FlakeRef, along with anything that might override that
+ * FlakeRef (like command-line overrides or "follows" specifications).
+ *
+ * A FlakeInput is also sometimes constructed directly from a FlakeRef
+ * instead of starting at the flake-file input specification
+ * (e.g. overrides, follows, and implicit inputs).
+ *
+ * A FlakeInput will usually have one of either "ref" or "follows"
+ * set.  If not otherwise specified, a "ref" will be generated to a
+ * 'type="indirect"' flake, which is treated as simply the name of a
+ * flake to be resolved in the registry.
+ */
 
 struct FlakeInput
 {
diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh
index 2596dd18e..0292eb210 100644
--- a/src/libexpr/flake/flakeref.hh
+++ b/src/libexpr/flake/flakeref.hh
@@ -12,32 +12,33 @@ class Store;
 
 typedef std::string FlakeId;
 
-// A flake reference specifies how to fetch a flake or raw source
-// (e.g. from a Git repository).  It is created from a URL-like syntax
-// (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
-// type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
-// path.
-//
-// Each flake will have a number of FlakeRef objects: one for each
-// input to the flake.
-//
-// The normal method of constructing a FlakeRef is by starting with an
-// input description (usually the attrs or a url from the flake file),
-// locating a fetcher for that input, and then capturing the Input
-// object that fetcher generates (usually via
-// FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
-//
-// The actual fetch not have been performed yet (i.e. a FlakeRef may
-// be lazy), but the fetcher can be invoked at any time via the
-// FlakeRef to ensure the store is populated with this input.
+/* A flake reference specifies how to fetch a flake or raw source
+ * (e.g. from a Git repository).  It is created from a URL-like syntax
+ * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
+ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
+ * path.
+ *
+ * Each flake will have a number of FlakeRef objects: one for each
+ * input to the flake.
+ *
+ * The normal method of constructing a FlakeRef is by starting with an
+ * input description (usually the attrs or a url from the flake file),
+ * locating a fetcher for that input, and then capturing the Input
+ * object that fetcher generates (usually via
+ * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
+ *
+ * The actual fetch not have been performed yet (i.e. a FlakeRef may
+ * be lazy), but the fetcher can be invoked at any time via the
+ * FlakeRef to ensure the store is populated with this input.
+ */
 
 struct FlakeRef
 {
-    // fetcher-specific representation of the input, sufficient to
-    // perform the fetch operation.
+    /* fetcher-specific representation of the input, sufficient to
+       perform the fetch operation. */
     fetchers::Input input;
 
-    // sub-path within the fetched input that represents this input
+    /* sub-path within the fetched input that represents this input */
     Path subdir;
 
     bool operator==(const FlakeRef & other) const;
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 6aa8a3422..84c954899 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -21,12 +21,13 @@ struct Tree
 
 struct InputScheme;
 
-// The Input object is generated by a specific fetcher, based on the
-// user-supplied input attribute in the flake.nix file, and contains
-// the information that the specific fetcher needs to perform the
-// actual fetch.  The Input object is most commonly created via the
-// "fromURL()" or "fromAttrs()" static functions which are provided the
-// url or attrset specified in the flake file.
+/* The Input object is generated by a specific fetcher, based on the
+ * user-supplied input attribute in the flake.nix file, and contains
+ * the information that the specific fetcher needs to perform the
+ * actual fetch.  The Input object is most commonly created via the
+ * "fromURL()" or "fromAttrs()" static functions which are provided
+ * the url or attrset specified in the flake file.
+ */
 
 struct Input
 {
@@ -90,13 +91,14 @@ public:
 };
 
 
-// The InputScheme represents a type of fetcher.  Each fetcher
-// registers with nix at startup time.  When processing an input for a
-// flake, each scheme is given an opportunity to "recognize" that
-// input from the url or attributes in the flake file's specification
-// and return an Input object to represent the input if it is
-// recognized.  The Input object contains the information the fetcher
-// needs to actually perform the "fetch()" when called.
+/* The InputScheme represents a type of fetcher.  Each fetcher
+ * registers with nix at startup time.  When processing an input for a
+ * flake, each scheme is given an opportunity to "recognize" that
+ * input from the url or attributes in the flake file's specification
+ * and return an Input object to represent the input if it is
+ * recognized.  The Input object contains the information the fetcher
+ * needs to actually perform the "fetch()" when called.
+ */
 
 struct InputScheme
 {

From c89fa3f644eec0652e97c11d9246f9580e5929fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 28 Sep 2020 21:08:14 +0300
Subject: [PATCH 308/882] Update .github/workflows/test.yml

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2f5c548d9..adc56e223 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,7 +13,6 @@ jobs:
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v11
-      with:
         skip_adding_nixpkgs_channel: true
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

From f1428484be248c7ba3f96379d8bf256b8df1a706 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 28 Sep 2020 21:08:24 +0300
Subject: [PATCH 309/882] Update .github/workflows/test.yml

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index adc56e223..829111b67 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,6 +13,5 @@ jobs:
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v11
-        skip_adding_nixpkgs_channel: true
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

From 00135e13f49e9b20e3ef03f2516e7cc277c40ca9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 28 Sep 2020 18:19:10 +0000
Subject: [PATCH 310/882] Clarify comment a bit

---
 src/libstore/derivations.hh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 74601134e..d48266774 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -128,11 +128,12 @@ struct Derivation : BasicDerivation
     std::string unparse(const Store & store, bool maskOutputs,
         std::map<std::string, StringSet> * actualInputs = nullptr) const;
 
-    /* Return the underlying basic derivation but with
+    /* Return the underlying basic derivation but with these changes:
 
-       1. input drv outputs moved to input sources.
+	   1. Input drvs are emptied, but the outputs of them that were used are
+	      added directly to input sources.
 
-       2. placeholders replaced with realized input store paths. */
+       2. Input placeholders are replaced with realized input store paths. */
     std::optional<BasicDerivation> tryResolve(Store & store);
 
     Derivation() = default;

From de86abbf3f9f0d18f7506b1d50072597786332ea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Sep 2020 12:55:06 +0200
Subject: [PATCH 311/882] Cleanup

---
 src/libfetchers/github.cc | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 7bf155c69..3734c4278 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -14,12 +14,6 @@ struct DownloadUrl
 {
     std::string url;
     std::optional<std::pair<std::string, std::string>> access_token_header;
-
-    DownloadUrl(const std::string & url)
-        : url(url) { }
-
-    DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
-        : url(url), access_token_header(access_token_header) { }
 };
 
 // A github or gitlab host
@@ -239,9 +233,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
         std::string accessToken = settings.githubAccessToken.get();
         if (accessToken != "") {
             auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
+            return DownloadUrl { url, auth_header };
         } else {
-            return DownloadUrl(url);
+            return DownloadUrl { url };
         }
     }
 
@@ -294,9 +288,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
         std::string accessToken = settings.gitlabAccessToken.get();
         if (accessToken != "") {
             auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
+            return DownloadUrl { url, auth_header };
         } else {
-            return DownloadUrl(url);
+            return DownloadUrl { url };
         }
 
     }

From 5999978a053b3ec16f448c40b54a1a62ceb82c90 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Sep 2020 13:05:19 +0200
Subject: [PATCH 312/882] Make Headers an optional argument

---
 src/libexpr/common-eval-args.cc  |  2 +-
 src/libexpr/parser.y             |  2 +-
 src/libexpr/primops/fetchTree.cc |  4 ++--
 src/libfetchers/fetchers.hh      |  8 ++++----
 src/libfetchers/github.cc        |  6 +++---
 src/libfetchers/registry.cc      |  2 +-
 src/libfetchers/tarball.cc       | 15 ++++++++-------
 src/libstore/filetransfer.hh     |  3 ---
 src/nix-channel/nix-channel.cc   |  6 +++---
 9 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index d71aa22f1..10c1a6975 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
     if (isUri(s)) {
         return state.store->toRealPath(
             fetchers::downloadTarball(
-                state.store, resolveUri(s), Headers {}, "source", false).first.storePath);
+                state.store, resolveUri(s), "source", false).first.storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index e879280c0..a4c84c526 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -718,7 +718,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
     if (isUri(elem.second)) {
         try {
             res = { true, store->toRealPath(fetchers::downloadTarball(
-                        store, resolveUri(elem.second), Headers {}, "source", false).first.storePath) };
+                        store, resolveUri(elem.second), "source", false).first.storePath) };
         } catch (FileTransferError & e) {
             logWarning({
                 .name = "Entry download",
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 3001957b4..06e8304b8 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -201,8 +201,8 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, Headers {}, name, (bool) expectedHash).first.storePath
-        : fetchers::downloadFile(state.store, *url, Headers{}, name, (bool) expectedHash).storePath;
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
+        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
     auto path = state.store->toRealPath(storePath);
 
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 0adc2c9f5..36d44f6e1 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -118,15 +118,15 @@ struct DownloadFileResult
 DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
-    const Headers & headers,
     const std::string & name,
-    bool immutable);
+    bool immutable,
+    const Headers & headers = {});
 
 std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
-    const Headers & headers,
     const std::string & name,
-    bool immutable);
+    bool immutable,
+    const Headers & headers = {});
 
 }
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 3734c4278..ec99481e1 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -175,7 +175,7 @@ struct GitArchiveInputScheme : InputScheme
             headers.push_back(*url.access_token_header);
         }
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
+        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, headers);
 
         input.attrs.insert_or_assign("lastModified", lastModified);
 
@@ -215,7 +215,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, headers, "source", false).storePath)));
+                    downloadFile(store, url, "source", false, headers).storePath)));
         auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
@@ -271,7 +271,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, headers, "source", false).storePath)));
+                    downloadFile(store, url, "source", false, headers).storePath)));
         auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 551e7684a..4367ee810 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -145,7 +145,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
         auto path = settings.flakeRegistry.get();
 
         if (!hasPrefix(path, "/")) {
-            auto storePath = downloadFile(store, path, Headers {}, "flake-registry.json", false).storePath;
+            auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
             path = store->toRealPath(storePath);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index cf6d6e3d2..ca49482a9 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -12,9 +12,9 @@ namespace nix::fetchers {
 DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
-    const Headers & headers,
     const std::string & name,
-    bool immutable)
+    bool immutable,
+    const Headers & headers)
 {
     // FIXME: check store
 
@@ -38,7 +38,8 @@ DownloadFileResult downloadFile(
     if (cached && !cached->expired)
         return useCached();
 
-    FileTransferRequest request(url, headers);
+    FileTransferRequest request(url);
+    request.headers = headers;
     if (cached)
         request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
     FileTransferResult res;
@@ -112,9 +113,9 @@ DownloadFileResult downloadFile(
 std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
-    const Headers & headers,
     const std::string & name,
-    bool immutable)
+    bool immutable,
+    const Headers & headers)
 {
     Attrs inAttrs({
         {"type", "tarball"},
@@ -130,7 +131,7 @@ std::pair<Tree, time_t> downloadTarball(
             getIntAttr(cached->infoAttrs, "lastModified")
         };
 
-    auto res = downloadFile(store, url, headers, name, immutable);
+    auto res = downloadFile(store, url, name, immutable, headers);
 
     std::optional<StorePath> unpackedStorePath;
     time_t lastModified;
@@ -225,7 +226,7 @@ struct TarballInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), Headers {}, "source", false).first;
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
         return {std::move(tree), input};
     }
 };
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index 7e302ff39..c89c51a21 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -66,9 +66,6 @@ struct FileTransferRequest
     FileTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
-    FileTransferRequest(const std::string & uri, Headers headers)
-        : uri(uri), headers(headers) { }
-
     std::string verb()
     {
         return data ? "upload" : "download";
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 760bbea86..e48f7af9a 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -94,7 +94,7 @@ static void update(const StringSet & channelNames)
         // We want to download the url to a file to see if it's a tarball while also checking if we
         // got redirected in the process, so that we can grab the various parts of a nix channel
         // definition from a consistent location if the redirect changes mid-download.
-        auto result = fetchers::downloadFile(store, url, Headers {}, std::string(baseNameOf(url)), false);
+        auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
         auto filename = store->toRealPath(result.storePath);
         url = result.effectiveUrl;
 
@@ -119,9 +119,9 @@ static void update(const StringSet & channelNames)
         if (!unpacked) {
             // Download the channel tarball.
             try {
-                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", Headers {}, "nixexprs.tar.xz", false).storePath);
+                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
             } catch (FileTransferError & e) {
-                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", Headers {}, "nixexprs.tar.bz2", false).storePath);
+                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
             }
         }
 

From 64e9b3c83b7cf7f3c7348426666ccca2ca395d28 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Sep 2020 23:33:16 +0200
Subject: [PATCH 313/882] nix registry list: Show 'dir' attribute

Issue #4050.
---
 src/libexpr/flake/flakeref.cc | 6 +++---
 src/libfetchers/fetchers.cc   | 8 ++++++++
 src/libfetchers/fetchers.hh   | 2 ++
 src/nix/registry.cc           | 4 ++--
 4 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
index d5c2ffe66..87b202643 100644
--- a/src/libexpr/flake/flakeref.cc
+++ b/src/libexpr/flake/flakeref.cc
@@ -16,10 +16,10 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege
 
 std::string FlakeRef::to_string() const
 {
-    auto url = input.toURL();
+    std::map<std::string, std::string> extraQuery;
     if (subdir != "")
-        url.query.insert_or_assign("dir", subdir);
-    return url.to_string();
+        extraQuery.insert_or_assign("dir", subdir);
+    return input.toURLString(extraQuery);
 }
 
 fetchers::Attrs FlakeRef::toAttrs() const
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index eaa635595..49851f7bc 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -69,6 +69,14 @@ ParsedURL Input::toURL() const
     return scheme->toURL(*this);
 }
 
+std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const
+{
+    auto url = toURL();
+    for (auto & attr : extraQuery)
+        url.query.insert(attr);
+    return url.to_string();
+}
+
 std::string Input::to_string() const
 {
     return toURL().to_string();
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 36d44f6e1..cc31a31b9 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -39,6 +39,8 @@ public:
 
     ParsedURL toURL() const;
 
+    std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const;
+
     std::string to_string() const;
 
     Attrs toAttrs() const;
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index 367268683..cb11ec195 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -31,8 +31,8 @@ struct CmdRegistryList : StoreCommand
                     registry->type == Registry::User   ? "user  " :
                     registry->type == Registry::System ? "system" :
                     "global",
-                    entry.from.to_string(),
-                    entry.to.to_string());
+                    entry.from.toURLString(),
+                    entry.to.toURLString(attrsToQuery(entry.extraAttrs)));
             }
         }
     }

From 5e7838512e2b8de3c8fe271b8beae5ca9e1efaf9 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Tue, 29 Sep 2020 16:20:54 -0700
Subject: [PATCH 314/882] Remove github-access-token in favor of access-token.

---
 src/libfetchers/github.cc | 10 ----------
 src/libstore/globals.hh   |  3 ---
 2 files changed, 13 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 142b8b87c..8286edf75 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -147,17 +147,7 @@ struct GitArchiveInputScheme : InputScheme
         auto tokens = settings.accessTokens.get();
         auto pat = tokens.find(host);
         if (pat == tokens.end())
-        {
-            if ("github.com" == host)
-            {
-                auto oldcfg = settings.githubAccessToken.get();
-                if (!oldcfg.empty()) {
-                    warn("using deprecated 'github-access-token' config value; please use 'access-tokens' instead");
-                    return oldcfg;
-                }
-            }
             return std::nullopt;
-        }
         return pat->second;
     }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3b8ccadf3..0f0c0fe6f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -859,9 +859,6 @@ public:
           are loaded as plugins (non-recursively).
         )"};
 
-    Setting<std::string> githubAccessToken{this, "", "github-access-token",
-        "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes (deprecated, please use 'access-tokens' instead)."};
-
     Setting<StringMap> accessTokens{this, {}, "access-tokens",
         R"(
           Access tokens used to access protected GitHub, GitLab, or

From 45a0ed82f089158a79c8c25ef844c55e4a74fc35 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 30 Sep 2020 00:39:06 +0000
Subject: [PATCH 315/882] Revert "Use template structs instead of phantoms"

This reverts commit 9ab07e99f527d1fa3adfa02839da477a1528d64b.
---
 src/libstore/build.cc            |   4 +-
 src/libstore/daemon.cc           |  38 ++++----
 src/libstore/derivations.cc      |   4 +-
 src/libstore/export-import.cc    |   4 +-
 src/libstore/legacy-ssh-store.cc |  14 +--
 src/libstore/remote-store.cc     |  70 +++++++-------
 src/libstore/worker-protocol.hh  | 151 ++++++++++++++++---------------
 src/nix-store/nix-store.cc       |  16 ++--
 8 files changed, 153 insertions(+), 148 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 56d454b6b..928858203 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1976,7 +1976,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    WorkerProto<StorePathSet>::write(worker.store, hook->sink, inputPaths);
+    nix::worker_proto::write(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1987,7 +1987,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known->isValid()) continue;
             missingPaths.insert(status.known->path);
         }
-        WorkerProto<StorePathSet>::write(worker.store, hook->sink, missingPaths);
+        nix::worker_proto::write(worker.store, hook->sink, missingPaths);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 98fd2048d..72203d1b2 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -247,7 +247,7 @@ static void writeValidPathInfo(
 {
     to << (info->deriver ? store->printStorePath(*info->deriver) : "")
        << info->narHash.to_string(Base16, false);
-    WorkerProto<StorePathSet>::write(*store, to, info->references);
+    nix::worker_proto::write(*store, to, info->references);
     to << info->registrationTime << info->narSize;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
         to << info->ultimate
@@ -272,11 +272,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = WorkerProto<StorePathSet>::read(*store, from);
+        auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto res = store->queryValidPaths(paths);
         logger->stopWork();
-        WorkerProto<StorePathSet>::write(*store, to, res);
+        nix::worker_proto::write(*store, to, res);
         break;
     }
 
@@ -292,11 +292,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = WorkerProto<StorePathSet>::read(*store, from);
+        auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        WorkerProto<StorePathSet>::write(*store, to, res);
+        nix::worker_proto::write(*store, to, res);
         break;
     }
 
@@ -325,7 +325,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        WorkerProto<StorePathSet>::write(*store, to, paths);
+        nix::worker_proto::write(*store, to, paths);
         break;
     }
 
@@ -343,7 +343,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        WorkerProto<std::map<std::string, std::optional<StorePath>>>::write(*store, to, outputs);
+        nix::worker_proto::write(*store, to, outputs);
         break;
     }
 
@@ -369,7 +369,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
             auto name = readString(from);
             auto camStr = readString(from);
-            auto refs = WorkerProto<StorePathSet>::read(*store, from);
+            auto refs = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
             bool repairBool;
             from >> repairBool;
             auto repair = RepairFlag{repairBool};
@@ -449,7 +449,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopAddTextToStore: {
         string suffix = readString(from);
         string s = readString(from);
-        auto refs = WorkerProto<StorePathSet>::read(*store, from);
+        auto refs = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -608,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
+        options.pathsToDelete = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -677,7 +677,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            WorkerProto<StorePathSet>::write(*store, to, i->second.references);
+            nix::worker_proto::write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -688,11 +688,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = WorkerProto<StorePathSet>::read(*store, from);
+            auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
+            pathsMap = nix::worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -700,7 +700,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            WorkerProto<StorePathSet>::write(*store, to, i.second.references);
+            nix::worker_proto::write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -710,7 +710,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        WorkerProto<StorePathSet>::write(*store, to, paths);
+        nix::worker_proto::write(*store, to, paths);
         break;
     }
 
@@ -782,7 +782,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.references = WorkerProto<StorePathSet>::read(*store, from);
+        info.references = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = parseContentAddressOpt(readString(from));
@@ -835,9 +835,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        WorkerProto<StorePathSet>::write(*store, to, willBuild);
-        WorkerProto<StorePathSet>::write(*store, to, willSubstitute);
-        WorkerProto<StorePathSet>::write(*store, to, unknown);
+        nix::worker_proto::write(*store, to, willBuild);
+        nix::worker_proto::write(*store, to, willSubstitute);
+        nix::worker_proto::write(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index faffe01e7..f8e7d773b 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -584,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
+    drv.inputSrcs = nix::worker_proto::read(store, in, Phantom<StorePathSet> {});
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -622,7 +622,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.output);
     }
-    WorkerProto<StorePathSet>::write(store, out, drv.inputSrcs);
+    nix::worker_proto::write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index b59b34dee..40a6f3c63 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    WorkerProto<StorePathSet>::write(*this, teeSink, info->references);
+    nix::worker_proto::write(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        auto references = WorkerProto<StorePathSet>::read(*this, source);
+        auto references = nix::worker_proto::read(*this, source, Phantom<StorePathSet> {});
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, *saved.s);
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index fdf3f91b9..e056859fd 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -122,7 +122,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
+            info->references = nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -156,7 +156,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
+            nix::worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -185,7 +185,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
+            nix::worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -301,10 +301,10 @@ public:
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
+        nix::worker_proto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
+        for (auto & i : nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
             out.insert(i);
     }
 
@@ -317,10 +317,10 @@ public:
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
+        nix::worker_proto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        return WorkerProto<StorePathSet>::read(*this, conn->from);
+        return nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 
     void connect() override
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 273466137..43853db4e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -24,61 +24,65 @@
 
 namespace nix {
 
-std::string WorkerProto<std::string>::read(const Store & store, Source & from)
+namespace worker_proto {
+
+std::string read(const Store & store, Source & from, Phantom<std::string> _)
 {
     return readString(from);
 }
 
-void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
+void write(const Store & store, Sink & out, const std::string & str)
 {
     out << str;
 }
 
 
-StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
 {
     return store.parseStorePath(readString(from));
 }
 
-void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
+void write(const Store & store, Sink & out, const StorePath & storePath)
 {
     out << store.printStorePath(storePath);
 }
 
 
-ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
+ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
 {
     return parseContentAddress(readString(from));
 }
 
-void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
+void write(const Store & store, Sink & out, const ContentAddress & ca)
 {
     out << renderContentAddress(ca);
 }
 
 
-std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
+std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
 {
     auto s = readString(from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
 {
     out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
+std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
 {
     return parseContentAddressOpt(readString(from));
 }
 
-void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
 {
     out << (caOpt ? renderContentAddress(*caOpt) : "");
 }
 
+}
+
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -325,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
+        nix::worker_proto::write(*this, conn->to, paths);
         conn.processStderr();
-        return WorkerProto<StorePathSet>::read(*this, conn->from);
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 }
 
@@ -337,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return WorkerProto<StorePathSet>::read(*this, conn->from);
+    return nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
@@ -354,9 +358,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        WorkerProto<StorePathSet>::write(*this, conn->to, paths);
+        nix::worker_proto::write(*this, conn->to, paths);
         conn.processStderr();
-        return WorkerProto<StorePathSet>::read(*this, conn->from);
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 }
 
@@ -378,7 +382,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -391,9 +395,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            WorkerProto<StorePathSet>::write(*this, conn->to, paths);
+            worker_proto::write(*this, conn->to, paths);
         } else
-            WorkerProto<StorePathCAMap>::write(*this, conn->to, pathsMap);
+            worker_proto::write(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -401,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -416,7 +420,7 @@ ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn,
     auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
     auto info = make_ref<ValidPathInfo>(path, narHash);
     if (deriver != "") info->deriver = parseStorePath(deriver);
-    info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
+    info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     conn->from >> info->registrationTime >> info->narSize;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
         conn->from >> info->ultimate;
@@ -460,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
+    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
         referrers.insert(i);
 }
 
@@ -470,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return WorkerProto<StorePathSet>::read(*this, conn->from);
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
@@ -482,7 +486,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return WorkerProto<StorePathSet>::read(*this, conn->from);
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
@@ -492,7 +496,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
         auto conn(getConnection());
         conn->to << wopQueryDerivationOutputMap << printStorePath(path);
         conn.processStderr();
-        return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
+        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
     } else {
         // Fallback for old daemon versions.
         // For floating-CA derivations (and their co-dependencies) this is an
@@ -537,7 +541,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
             << wopAddToStore
             << name
             << renderContentAddressMethod(caMethod);
-        WorkerProto<StorePathSet>::write(*this, conn->to, references);
+        worker_proto::write(*this, conn->to, references);
         conn->to << repair;
 
         conn.withFramedSink([&](Sink & sink) {
@@ -554,7 +558,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
             [&](TextHashMethod thm) -> void {
                 std::string s = dump.drain();
                 conn->to << wopAddTextToStore << name << s;
-                WorkerProto<StorePathSet>::write(*this, conn->to, references);
+                worker_proto::write(*this, conn->to, references);
                 conn.processStderr();
             },
             [&](FixedOutputHashMethod fohm) -> void {
@@ -623,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            WorkerProto<StorePathSet>::write(*this, sink, info.references);
+            worker_proto::write(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -633,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
+        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
         assert(importedPaths.size() <= 1);
     }
 
@@ -642,7 +646,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
+        worker_proto::write(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -764,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << wopCollectGarbage << options.action;
-    WorkerProto<StorePathSet>::write(*this, conn->to, options.pathsToDelete);
+    worker_proto::write(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -826,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
             ss.push_back(p.to_string(*this));
         conn->to << ss;
         conn.processStderr();
-        willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
-        willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
-        unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
+        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
         conn->from >> downloadSize >> narSize;
         return;
     }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index bb87cf3ec..fd6c2b2cf 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -66,96 +66,95 @@ typedef enum {
 class Store;
 struct Source;
 
+/* To guide overloading */
 template<typename T>
-struct WorkerProto {
-    static T read(const Store & store, Source & from);
-    static void write(const Store & store, Sink & out, const T & t);
-};
+struct Phantom {};
 
-#define MAKE_WORKER_PROTO(T) \
-    template<> \
-    struct WorkerProto< T > { \
-        static T read(const Store & store, Source & from); \
-        static void write(const Store & store, Sink & out, const T & t); \
-    }
 
-MAKE_WORKER_PROTO(std::string);
-MAKE_WORKER_PROTO(StorePath);
-MAKE_WORKER_PROTO(ContentAddress);
+namespace worker_proto {
+/* FIXME maybe move more stuff inside here */
+
+#define MAKE_WORKER_PROTO(TEMPLATE, T) \
+    TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
+    TEMPLATE void write(const Store & store, Sink & out, const T & str)
+
+MAKE_WORKER_PROTO(, std::string);
+MAKE_WORKER_PROTO(, StorePath);
+MAKE_WORKER_PROTO(, ContentAddress);
+
+MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
+MAKE_WORKER_PROTO(template<typename T>, std::optional<T>);
+
+#define X_ template<typename K, typename V>
+#define Y_ std::map<K, V>
+MAKE_WORKER_PROTO(X_, Y_);
+#undef X_
+#undef Y_
 
 template<typename T>
-struct WorkerProto<std::set<T>> {
-
-    static std::set<T> read(const Store & store, Source & from)
-    {
-        std::set<T> resSet;
-        auto size = readNum<size_t>(from);
-        while (size--) {
-            resSet.insert(WorkerProto<T>::read(store, from));
-        }
-        return resSet;
+std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
+{
+    std::set<T> resSet;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        resSet.insert(read(store, from, Phantom<T> {}));
     }
+    return resSet;
+}
 
-    static void write(const Store & store, Sink & out, const std::set<T> & resSet)
-    {
-        out << resSet.size();
-        for (auto & key : resSet) {
-            WorkerProto<T>::write(store, out, key);
-        }
+template<typename T>
+void write(const Store & store, Sink & out, const std::set<T> & resSet)
+{
+    out << resSet.size();
+    for (auto & key : resSet) {
+        write(store, out, key);
     }
-
-};
+}
 
 template<typename K, typename V>
-struct WorkerProto<std::map<K, V>> {
-
-    static std::map<K, V> read(const Store & store, Source & from)
-    {
-        std::map<K, V> resMap;
-        auto size = readNum<size_t>(from);
-        while (size--) {
-            auto k = WorkerProto<K>::read(store, from);
-            auto v = WorkerProto<V>::read(store, from);
-            resMap.insert_or_assign(std::move(k), std::move(v));
-        }
-        return resMap;
+std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
+{
+    std::map<K, V> resMap;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        auto k = read(store, from, Phantom<K> {});
+        auto v = read(store, from, Phantom<V> {});
+        resMap.insert_or_assign(std::move(k), std::move(v));
     }
+    return resMap;
+}
 
-    static void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
-    {
-        out << resMap.size();
-        for (auto & i : resMap) {
-            WorkerProto<K>::write(store, out, i.first);
-            WorkerProto<V>::write(store, out, i.second);
-        }
+template<typename K, typename V>
+void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+{
+    out << resMap.size();
+    for (auto & i : resMap) {
+        write(store, out, i.first);
+        write(store, out, i.second);
     }
-
-};
+}
 
 template<typename T>
-struct WorkerProto<std::optional<T>> {
-
-    static std::optional<T> read(const Store & store, Source & from)
-    {
-        auto tag = readNum<uint8_t>(from);
-        switch (tag) {
-        case 0:
-            return std::nullopt;
-        case 1:
-            return WorkerProto<T>::read(store, from);
-        default:
-            throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
-        }
+std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _)
+{
+    auto tag = readNum<uint8_t>(from);
+    switch (tag) {
+    case 0:
+        return std::nullopt;
+    case 1:
+        return read(store, from, Phantom<T> {});
+    default:
+        throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
     }
+}
 
-    static void write(const Store & store, Sink & out, const std::optional<T> & optVal)
-    {
-        out << (uint64_t) (optVal ? 1 : 0);
-        if (optVal)
-            WorkerProto<T>::write(store, out, *optVal);
-    }
-
-};
+template<typename T>
+void write(const Store & store, Sink & out, const std::optional<T> & optVal)
+{
+    out << (uint64_t) (optVal ? 1 : 0);
+    if (optVal)
+        nix::worker_proto::write(store, out, *optVal);
+}
 
 /* Specialization which uses and empty string for the empty case, taking
    advantage of the fact these types always serialize to non-empty strings.
@@ -163,7 +162,9 @@ struct WorkerProto<std::optional<T>> {
    std::optional<T>, where <= is the compatability partial order, T is one of
    the types below.
  */
-MAKE_WORKER_PROTO(std::optional<StorePath>);
-MAKE_WORKER_PROTO(std::optional<ContentAddress>);
+MAKE_WORKER_PROTO(, std::optional<StorePath>);
+MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
+
+}
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 141dab478..4dcdebe2f 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -822,7 +822,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = WorkerProto<StorePathSet>::read(*store, in);
+                auto paths = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -852,19 +852,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                         }
                 }
 
-                WorkerProto<StorePathSet>::write(*store, out, store->queryValidPaths(paths));
+                nix::worker_proto::write(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                auto paths = WorkerProto<StorePathSet>::read(*store, in);
+                auto paths = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        WorkerProto<StorePathSet>::write(*store, out, info->references);
+                        nix::worker_proto::write(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(WorkerProto<StorePathSet>::read(*store, in), out);
+                store->exportPaths(nix::worker_proto::read(*store, in, Phantom<StorePathSet> {}), out);
                 break;
             }
 
@@ -941,9 +941,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(WorkerProto<StorePathSet>::read(*store, in),
+                store->computeFSClosure(nix::worker_proto::read(*store, in, Phantom<StorePathSet> {}),
                     closure, false, includeOutputs);
-                WorkerProto<StorePathSet>::write(*store, out, closure);
+                nix::worker_proto::write(*store, out, closure);
                 break;
             }
 
@@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = WorkerProto<StorePathSet>::read(*store, in);
+                info.references = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = parseContentAddressOpt(readString(in));

From b7597016529cebdc3c9432a101c1f8d9227713cc Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 30 Sep 2020 00:41:18 +0000
Subject: [PATCH 316/882] nix::worker_proto -> worker_proto

---
 src/libstore/build.cc            |  4 ++--
 src/libstore/daemon.cc           | 38 ++++++++++++++++----------------
 src/libstore/derivations.cc      |  4 ++--
 src/libstore/export-import.cc    |  4 ++--
 src/libstore/legacy-ssh-store.cc | 14 ++++++------
 src/libstore/remote-store.cc     |  6 ++---
 src/libstore/worker-protocol.hh  |  2 +-
 src/nix-store/nix-store.cc       | 16 +++++++-------
 8 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 928858203..c688acd58 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1976,7 +1976,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    nix::worker_proto::write(worker.store, hook->sink, inputPaths);
+    worker_proto::write(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1987,7 +1987,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known->isValid()) continue;
             missingPaths.insert(status.known->path);
         }
-        nix::worker_proto::write(worker.store, hook->sink, missingPaths);
+        worker_proto::write(worker.store, hook->sink, missingPaths);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 72203d1b2..0713c4853 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -247,7 +247,7 @@ static void writeValidPathInfo(
 {
     to << (info->deriver ? store->printStorePath(*info->deriver) : "")
        << info->narHash.to_string(Base16, false);
-    nix::worker_proto::write(*store, to, info->references);
+    worker_proto::write(*store, to, info->references);
     to << info->registrationTime << info->narSize;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
         to << info->ultimate
@@ -272,11 +272,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto res = store->queryValidPaths(paths);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, res);
+        worker_proto::write(*store, to, res);
         break;
     }
 
@@ -292,11 +292,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, res);
+        worker_proto::write(*store, to, res);
         break;
     }
 
@@ -325,7 +325,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, paths);
+        worker_proto::write(*store, to, paths);
         break;
     }
 
@@ -343,7 +343,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, outputs);
+        worker_proto::write(*store, to, outputs);
         break;
     }
 
@@ -369,7 +369,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
             auto name = readString(from);
             auto camStr = readString(from);
-            auto refs = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
             bool repairBool;
             from >> repairBool;
             auto repair = RepairFlag{repairBool};
@@ -449,7 +449,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopAddTextToStore: {
         string suffix = readString(from);
         string s = readString(from);
-        auto refs = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -608,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -677,7 +677,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            nix::worker_proto::write(*store, to, i->second.references);
+            worker_proto::write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -688,11 +688,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = nix::worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
+            pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -700,7 +700,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            nix::worker_proto::write(*store, to, i.second.references);
+            worker_proto::write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -710,7 +710,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        nix::worker_proto::write(*store, to, paths);
+        worker_proto::write(*store, to, paths);
         break;
     }
 
@@ -782,7 +782,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.references = nix::worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = parseContentAddressOpt(readString(from));
@@ -835,9 +835,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        nix::worker_proto::write(*store, to, willBuild);
-        nix::worker_proto::write(*store, to, willSubstitute);
-        nix::worker_proto::write(*store, to, unknown);
+        worker_proto::write(*store, to, willBuild);
+        worker_proto::write(*store, to, willSubstitute);
+        worker_proto::write(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index f8e7d773b..7ffc94818 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -584,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = nix::worker_proto::read(store, in, Phantom<StorePathSet> {});
+    drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -622,7 +622,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.output);
     }
-    nix::worker_proto::write(store, out, drv.inputSrcs);
+    worker_proto::write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 40a6f3c63..02c839520 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    nix::worker_proto::write(*this, teeSink, info->references);
+    worker_proto::write(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        auto references = nix::worker_proto::read(*this, source, Phantom<StorePathSet> {});
+        auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, *saved.s);
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index e056859fd..6db44047d 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -122,7 +122,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -156,7 +156,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            nix::worker_proto::write(*this, conn->to, info.references);
+            worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -185,7 +185,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            nix::worker_proto::write(*this, conn->to, info.references);
+            worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -301,10 +301,10 @@ public:
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        nix::worker_proto::write(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+        for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
             out.insert(i);
     }
 
@@ -317,10 +317,10 @@ public:
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        nix::worker_proto::write(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        return nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 
     void connect() override
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 43853db4e..049d4e954 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -329,7 +329,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        nix::worker_proto::write(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn.processStderr();
         return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
@@ -341,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return nix::worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
@@ -358,7 +358,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        nix::worker_proto::write(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn.processStderr();
         return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index fd6c2b2cf..2934c1d67 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -153,7 +153,7 @@ void write(const Store & store, Sink & out, const std::optional<T> & optVal)
 {
     out << (uint64_t) (optVal ? 1 : 0);
     if (optVal)
-        nix::worker_proto::write(store, out, *optVal);
+        worker_proto::write(store, out, *optVal);
 }
 
 /* Specialization which uses and empty string for the empty case, taking
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 4dcdebe2f..da91caef1 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -822,7 +822,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -852,19 +852,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                         }
                 }
 
-                nix::worker_proto::write(*store, out, store->queryValidPaths(paths));
+                worker_proto::write(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                auto paths = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        nix::worker_proto::write(*store, out, info->references);
+                        worker_proto::write(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(nix::worker_proto::read(*store, in, Phantom<StorePathSet> {}), out);
+                store->exportPaths(worker_proto::read(*store, in, Phantom<StorePathSet> {}), out);
                 break;
             }
 
@@ -941,9 +941,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(nix::worker_proto::read(*store, in, Phantom<StorePathSet> {}),
+                store->computeFSClosure(worker_proto::read(*store, in, Phantom<StorePathSet> {}),
                     closure, false, includeOutputs);
-                nix::worker_proto::write(*store, out, closure);
+                worker_proto::write(*store, out, closure);
                 break;
             }
 
@@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = nix::worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = parseContentAddressOpt(readString(in));

From 274357eb6acd9a0812872f32dd487354d51f0f13 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 Sep 2020 12:09:18 +0200
Subject: [PATCH 317/882] Simplify

---
 src/libfetchers/github.cc | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8286edf75..42eb346a5 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -15,9 +15,6 @@ struct DownloadUrl
 {
     std::string url;
     Headers headers;
-
-    DownloadUrl(const std::string & url, const Headers & headers)
-        : url(url), headers(headers) { }
 };
 
 // A github or gitlab host
@@ -254,7 +251,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             input.getRev()->to_string(Base16, false));
 
         Headers headers = makeHeadersWithAuthTokens(host);
-        return DownloadUrl(url, headers);
+        return DownloadUrl { url, headers };
     }
 
     void clone(const Input & input, const Path & destDir) override
@@ -320,7 +317,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             input.getRev()->to_string(Base16, false));
 
         Headers headers = makeHeadersWithAuthTokens(host);
-        return DownloadUrl(url, headers);
+        return DownloadUrl { url, headers };
     }
 
     void clone(const Input & input, const Path & destDir) override

From 20a1e20d9194527d725898c745d1243d3de16277 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 Sep 2020 12:11:22 +0200
Subject: [PATCH 318/882] Style

---
 src/libfetchers/github.cc | 19 +++++++++++--------
 src/libstore/globals.hh   |  8 ++++----
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 42eb346a5..8610fe447 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -140,15 +140,16 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    std::optional<std::string> getAccessToken(const std::string &host) const {
+    std::optional<std::string> getAccessToken(const std::string & host) const
+    {
         auto tokens = settings.accessTokens.get();
-        auto pat = tokens.find(host);
-        if (pat == tokens.end())
-            return std::nullopt;
-        return pat->second;
+        if (auto token = get(tokens, host))
+            return *token;
+        return {};
     }
 
-    Headers makeHeadersWithAuthTokens(const std::string & host) const {
+    Headers makeHeadersWithAuthTokens(const std::string & host) const
+    {
         Headers headers;
         auto accessToken = getAccessToken(host);
         if (accessToken) {
@@ -214,7 +215,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const
+    {
         // Github supports PAT/OAuth2 tokens and HTTP Basic
         // Authentication.  The former simply specifies the token, the
         // latter can use the token as the password.  Only the first
@@ -268,7 +270,8 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
-    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const
+    {
         // Gitlab supports 4 kinds of authorization, two of which are
         // relevant here: OAuth2 and PAT (Private Access Token).  The
         // user can indicate which token is used by specifying the
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0f0c0fe6f..8c63c5b34 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -893,10 +893,10 @@ public:
 
           ```nix
           input.foo = {
-            type="gitlab";
-            host="gitlab.mycompany.com";
-            owner="mycompany";
-            repo="pro";
+            type = "gitlab";
+            host = "gitlab.mycompany.com";
+            owner = "mycompany";
+            repo = "pro";
           };
           ```
 

From 924712eef1fe86d349635ba666d413632e62519c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 Sep 2020 17:48:49 +0200
Subject: [PATCH 319/882] Installer: Set a known umask

Fixes #1560, #2377.
---
 scripts/install-nix-from-closure.sh | 2 ++
 scripts/install.in                  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 6efd8af18..14fb91534 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+umask 0022
+
 dest="/nix"
 self="$(dirname "$0")"
 nix="@nix@"
diff --git a/scripts/install.in b/scripts/install.in
index 1d26c4ff0..39fae37e3 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -10,6 +10,8 @@ oops() {
     exit 1
 }
 
+umask 0022
+
 tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
           oops "Can't create temporary directory for downloading the Nix binary tarball")"
 cleanup() {

From f3280004e2be3f7533190ddf71ee1ec7c0d7d60d Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Thu, 1 Oct 2020 11:34:13 +0700
Subject: [PATCH 320/882] add more examples to --help of `nix run`

---
 src/nix/run.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index cbaba9d90..e6584346e 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -167,6 +167,14 @@ struct CmdRun : InstallableCommand, RunCommon
                 "To run Blender:",
                 "nix run blender-bin"
             },
+            Example{
+                "To run vim from nixpkgs:",
+                "nix run nixpkgs#vim"
+            },
+            Example{
+                "To run vim from nixpkgs with arguments:",
+                "nix run nixpkgs#vim -- --help"
+            },
         };
     }
 

From d5d196b0a15fdba50363b89a61eb894a49153b08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabian=20M=C3=B6ller?= <fabianm88@gmail.com>
Date: Fri, 2 Oct 2020 12:10:31 +0200
Subject: [PATCH 321/882] Fix profile update in nix command

---
 src/nix/command.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 37a4bc785..ba7de9fdd 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -152,7 +152,7 @@ void MixProfile::updateProfile(const Buildables & buildables)
                 for (auto & output : bfd.outputs) {
                     /* Output path should be known because we just tried to
                        build it. */
-                    assert(!output.second);
+                    assert(output.second);
                     result.push_back(*output.second);
                 }
             },

From 4abd5554ad0be209b103378a26fa8784aee3a0a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabian=20M=C3=B6ller?= <fabianm88@gmail.com>
Date: Fri, 2 Oct 2020 15:40:55 +0200
Subject: [PATCH 322/882] Fix macOS sandbox build

---
 doc/manual/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 3b8e7e2df..d308c7cf6 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -40,7 +40,7 @@ $(d)/nix.json: $(bindir)/nix
 	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix

From 88a667e49e10af4a9e2daa51badbed63ad19d817 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Oct 2020 17:53:30 +0200
Subject: [PATCH 323/882] Fix s3:// store

Fixes https://github.com/NixOS/nixos-org-configurations/issues/123.
---
 src/libstore/http-binary-cache-store.cc | 1 +
 src/libstore/store-api.hh               | 1 +
 src/libutil/url-parts.hh                | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 86be7c006..3d3d91e5e 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -73,6 +73,7 @@ public:
         if (forceHttp) ret.insert("file");
         return ret;
     }
+
 protected:
 
     void maybeDisable()
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ba202375b..854446987 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -830,6 +830,7 @@ struct StoreFactory
     std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
     std::function<std::shared_ptr<StoreConfig> ()> getConfig;
 };
+
 struct Implementations
 {
     static std::vector<StoreFactory> * registered;
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 64e06cfbc..68be15cb0 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -7,7 +7,7 @@ namespace nix {
 
 // URI stuff.
 const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
-const static std::string schemeRegex = "(?:[a-z+.-]+)";
+const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
 const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
 const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
 const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";

From d0bb544128140d51b5a5a17d36eeb9fd13838586 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 10:18:44 +0200
Subject: [PATCH 324/882] Add missing #pragma once

---
 src/libexpr/primops.hh             | 2 ++
 src/libstore/daemon.hh             | 2 ++
 src/libstore/parsed-derivations.hh | 2 ++
 src/libutil/topo-sort.hh           | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index ed5e2ea58..9d42d6539 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -1,3 +1,5 @@
+#pragma once
+
 #include "eval.hh"
 
 #include <tuple>
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
index 841ace316..67755d54e 100644
--- a/src/libstore/daemon.hh
+++ b/src/libstore/daemon.hh
@@ -1,3 +1,5 @@
+#pragma once
+
 #include "serialise.hh"
 #include "store-api.hh"
 
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 3fa09f34f..c9fbe68c4 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -1,3 +1,5 @@
+#pragma once
+
 #include "store-api.hh"
 
 #include <nlohmann/json_fwd.hpp>
diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh
index 7a68ff169..7418be5e0 100644
--- a/src/libutil/topo-sort.hh
+++ b/src/libutil/topo-sort.hh
@@ -1,3 +1,5 @@
+#pragma once
+
 #include "error.hh"
 
 namespace nix {

From 6691256e790b2917ccd06f16472b667426d0f413 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 10:40:49 +0200
Subject: [PATCH 325/882] Factor out common showBytes()

---
 src/libmain/shared.cc          | 12 +++---------
 src/libstore/optimise-store.cc | 12 +++---------
 src/libutil/util.cc            |  7 +++++++
 src/libutil/util.hh            |  3 +++
 4 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 22ae51e47..2247aeca4 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -386,18 +386,12 @@ RunPager::~RunPager()
 }
 
 
-string showBytes(uint64_t bytes)
-{
-    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
-}
-
-
 PrintFreed::~PrintFreed()
 {
     if (show)
-        std::cout << format("%1% store paths deleted, %2% freed\n")
-            % results.paths.size()
-            % showBytes(results.bytesFreed);
+        std::cout << fmt("%d store paths deleted, %s freed\n",
+            results.paths.size(),
+            showBytes(results.bytesFreed));
 }
 
 Exit::~Exit() { }
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index c032a5e22..a0d482ddf 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -276,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
     }
 }
 
-static string showBytes(uint64_t bytes)
-{
-    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
-}
-
 void LocalStore::optimiseStore()
 {
     OptimiseStats stats;
 
     optimiseStore(stats);
 
-    printInfo(
-        format("%1% freed by hard-linking %2% files")
-        % showBytes(stats.bytesFreed)
-        % stats.filesLinked);
+    printInfo("%s freed by hard-linking %d files",
+        showBytes(stats.bytesFreed),
+        stats.filesLinked);
 }
 
 void LocalStore::optimisePath(const Path & path)
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 9e7142e01..f07e99885 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1641,4 +1641,11 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
     return fdSocket;
 }
 
+
+string showBytes(uint64_t bytes)
+{
+    return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));
+}
+
+
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b8e201203..129d59a97 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -573,4 +573,7 @@ template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
 template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
 
+std::string showBytes(uint64_t bytes);
+
+
 }

From 636455c471bfde89dbdf33e10308962b79d50f07 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 11:16:32 +0200
Subject: [PATCH 326/882] Remove 'using namespace fetchers'

---
 src/libexpr/flake/flake.cc    | 2 +-
 src/libexpr/flake/lockfile.cc | 4 ++--
 src/libexpr/flake/lockfile.hh | 2 --
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index b4ede542c..bae4d65e5 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -12,7 +12,7 @@ using namespace flake;
 
 namespace flake {
 
-typedef std::pair<Tree, FlakeRef> FetchedFlake;
+typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake;
 typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
 
 static std::optional<FetchedFlake> lookupInFlakeCache(
diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index 78431f000..bb46e1bb4 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -13,12 +13,12 @@ FlakeRef getFlakeRef(
 {
     auto i = json.find(attr);
     if (i != json.end()) {
-        auto attrs = jsonToAttrs(*i);
+        auto attrs = fetchers::jsonToAttrs(*i);
         // FIXME: remove when we drop support for version 5.
         if (info) {
             auto j = json.find(info);
             if (j != json.end()) {
-                for (auto k : jsonToAttrs(*j))
+                for (auto k : fetchers::jsonToAttrs(*j))
                     attrs.insert_or_assign(k.first, k.second);
             }
         }
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
index 9ec8b39c3..627794d8c 100644
--- a/src/libexpr/flake/lockfile.hh
+++ b/src/libexpr/flake/lockfile.hh
@@ -11,8 +11,6 @@ class StorePath;
 
 namespace nix::flake {
 
-using namespace fetchers;
-
 typedef std::vector<FlakeId> InputPath;
 
 struct LockedNode;

From b4db315a56860b10a7d19339f2310bc1149219cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 13:26:00 +0200
Subject: [PATCH 327/882] mk/precompiled-headers.mk: Fix clang test

"clang++" includes the string "g++" so this test didn't work
properly. However the separate handling of clang might not be needed
anymore...
---
 mk/precompiled-headers.mk | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index 1fdb4b3a4..bfee35a2c 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -21,18 +21,18 @@ clean-files += $(GCH) $(PCH)
 
 ifeq ($(PRECOMPILE_HEADERS), 1)
 
-  ifeq ($(findstring g++,$(CXX)), g++)
-
-    GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
-
-    GLOBAL_ORDER_AFTER += $(GCH)
-
-  else ifeq ($(findstring clang++,$(CXX)), clang++)
+  ifeq ($(findstring clang++,$(CXX)), clang++)
 
     GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
 
     GLOBAL_ORDER_AFTER += $(PCH)
 
+  else ifeq ($(findstring g++,$(CXX)), g++)
+
+    GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(GCH)
+
   else
 
     $(error Don't know how to precompile headers on $(CXX))

From 0856c0a0b4ff9b72b4e786e700785bb331298582 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 13:27:09 +0200
Subject: [PATCH 328/882] mk/precompiled-headers.mk: Remove special handling
 for clang

---
 .gitignore                |  1 -
 local.mk                  |  2 +-
 mk/precompiled-headers.mk | 27 +++------------------------
 3 files changed, 4 insertions(+), 26 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4711fa7fa..b087cd8d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,7 +5,6 @@ perl/Makefile.config
 /aclocal.m4
 /autom4te.cache
 /precompiled-headers.h.gch
-/precompiled-headers.h.pch
 /config.*
 /configure
 /stamp-h1
diff --git a/local.mk b/local.mk
index b1ca832a6..fe314b902 100644
--- a/local.mk
+++ b/local.mk
@@ -11,6 +11,6 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 $(foreach i, config.h $(wildcard src/lib*/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
-$(GCH) $(PCH): src/libutil/util.hh config.h
+$(GCH): src/libutil/util.hh config.h
 
 GCH_CXXFLAGS = -I src/libutil
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index bfee35a2c..cdd3daecd 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -10,33 +10,12 @@ $(GCH): precompiled-headers.h
 	@mkdir -p "$(dir $@)"
 	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
-PCH = $(buildprefix)precompiled-headers.h.pch
-
-$(PCH): precompiled-headers.h
-	@rm -f $@
-	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
-
-clean-files += $(GCH) $(PCH)
+clean-files += $(GCH)
 
 ifeq ($(PRECOMPILE_HEADERS), 1)
 
-  ifeq ($(findstring clang++,$(CXX)), clang++)
+  GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
 
-    GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
-
-    GLOBAL_ORDER_AFTER += $(PCH)
-
-  else ifeq ($(findstring g++,$(CXX)), g++)
-
-    GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
-
-    GLOBAL_ORDER_AFTER += $(GCH)
-
-  else
-
-    $(error Don't know how to precompile headers on $(CXX))
-
-  endif
+  GLOBAL_ORDER_AFTER += $(GCH)
 
 endif

From 0419cd26954d9d7c5b5ace96501b41cc81d66eed Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 13:34:58 +0200
Subject: [PATCH 329/882] Remove unneeded -lboost_* flags

---
 src/nix/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/local.mk b/src/nix/local.mk
index ab4e9121b..f37b73384 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -19,7 +19,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -llowdown
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \

From 85c8be6286e8f5464813a8a29b0b78d892498745 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 13:36:55 +0200
Subject: [PATCH 330/882] Remove static variable name clashes

This was useful for an experiment with building Nix as a single
compilation unit. It's not very useful otherwise but also doesn't
hurt...
---
 src/build-remote/build-remote.cc               |  4 ++--
 src/libexpr/eval.cc                            |  2 +-
 src/libexpr/primops/context.cc                 | 10 +++++-----
 src/libexpr/primops/fetchMercurial.cc          |  2 +-
 src/libexpr/primops/fetchTree.cc               |  2 +-
 src/libexpr/primops/fromTOML.cc                |  2 +-
 src/libfetchers/git.cc                         |  2 +-
 src/libfetchers/github.cc                      |  4 ++--
 src/libfetchers/indirect.cc                    |  2 +-
 src/libfetchers/mercurial.cc                   |  2 +-
 src/libfetchers/path.cc                        |  2 +-
 src/libfetchers/tarball.cc                     |  2 +-
 src/libstore/dummy-store.cc                    |  2 +-
 src/libstore/filetransfer.cc                   |  2 +-
 src/libstore/globals.cc                        |  2 +-
 src/libstore/http-binary-cache-store.cc        |  2 +-
 src/libstore/legacy-ssh-store.cc               |  2 +-
 src/libstore/local-binary-cache-store.cc       |  2 +-
 src/libstore/remote-store.cc                   |  2 +-
 src/libstore/s3-binary-cache-store.cc          |  2 +-
 src/libstore/ssh-store.cc                      |  2 +-
 src/libutil/archive.cc                         |  2 +-
 src/libutil/args.cc                            |  6 +++---
 src/libutil/logging.cc                         |  2 +-
 src/nix-build/nix-build.cc                     |  6 +++---
 src/nix-channel/nix-channel.cc                 |  4 ++--
 src/nix-collect-garbage/nix-collect-garbage.cc |  4 ++--
 src/nix-copy-closure/nix-copy-closure.cc       |  4 ++--
 src/nix-daemon/nix-daemon.cc                   |  4 ++--
 src/nix-env/nix-env.cc                         |  4 ++--
 src/nix-instantiate/nix-instantiate.cc         |  4 ++--
 src/nix-prefetch-url/nix-prefetch-url.cc       |  4 ++--
 src/nix-store/nix-store.cc                     |  9 +++++++--
 src/nix/add-to-store.cc                        |  2 +-
 src/nix/build.cc                               |  2 +-
 src/nix/cat.cc                                 |  4 ++--
 src/nix/copy.cc                                |  2 +-
 src/nix/describe-stores.cc                     |  2 +-
 src/nix/develop.cc                             |  4 ++--
 src/nix/diff-closures.cc                       |  2 +-
 src/nix/doctor.cc                              |  2 +-
 src/nix/dump-path.cc                           |  2 +-
 src/nix/edit.cc                                |  2 +-
 src/nix/eval.cc                                |  2 +-
 src/nix/flake.cc                               |  2 +-
 src/nix/hash.cc                                | 14 +++++++-------
 src/nix/log.cc                                 |  2 +-
 src/nix/ls.cc                                  |  4 ++--
 src/nix/make-content-addressable.cc            |  2 +-
 src/nix/optimise-store.cc                      |  2 +-
 src/nix/path-info.cc                           |  2 +-
 src/nix/ping-store.cc                          |  2 +-
 src/nix/profile.cc                             |  2 +-
 src/nix/registry.cc                            |  2 +-
 src/nix/repl.cc                                |  2 +-
 src/nix/run.cc                                 |  4 ++--
 src/nix/search.cc                              |  2 +-
 src/nix/show-config.cc                         |  2 +-
 src/nix/show-derivation.cc                     |  2 +-
 src/nix/sigs.cc                                |  4 ++--
 src/nix/upgrade-nix.cc                         |  2 +-
 src/nix/verify.cc                              |  2 +-
 src/nix/why-depends.cc                         |  2 +-
 63 files changed, 97 insertions(+), 92 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index ce5127113..9a0e9f08d 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -44,7 +44,7 @@ static bool allSupportedLocally(Store & store, const std::set<std::string>& requ
     return true;
 }
 
-static int _main(int argc, char * * argv)
+static int main_build_remote(int argc, char * * argv)
 {
     {
         logger = makeJSONLogger(*logger);
@@ -297,4 +297,4 @@ connected:
     }
 }
 
-static RegisterLegacyCommand s1("build-remote", _main);
+static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 883fc27a7..d6366050c 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2081,7 +2081,7 @@ Strings EvalSettings::getDefaultNixPath()
 
 EvalSettings evalSettings;
 
-static GlobalConfig::Register r1(&evalSettings);
+static GlobalConfig::Register rEvalSettings(&evalSettings);
 
 
 }
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index dbb93bae6..b570fca31 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos,
     mkString(v, s, PathSet());
 }
 
-static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
 
 
 static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
     mkBool(v, !context.empty());
 }
 
-static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
 
 
 /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
@@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
     mkString(v, s, context2);
 }
 
-static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
 
 
 /* Extract the context of a string as a structured Nix value.
@@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
     v.attrs->sort();
 }
 
-static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
 
 
 /* Append the given context to a given string.
@@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     mkString(v, orig, context);
 }
 
-static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);
 
 }
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 1a064ed5c..a77035c16 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -87,6 +87,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
         state.allowedPaths->insert(tree.actualPath);
 }
 
-static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
+static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);
 
 }
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 06e8304b8..7cd4d0fbf 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -152,7 +152,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
     fetchTree(state, pos, args, v, std::nullopt);
 }
 
-static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
 
 static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     const string & who, bool unpack, std::string name)
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index b00827a4b..77bff44ae 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -88,6 +88,6 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     }
 }
 
-static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
+static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML);
 
 }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index ad7638d73..a6411b02b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -452,6 +452,6 @@ struct GitInputScheme : InputScheme
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
 
 }
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8610fe447..90893075e 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -334,7 +334,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
-static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
+static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
 
 }
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 74332ae3d..10e59919a 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -100,6 +100,6 @@ struct IndirectInputScheme : InputScheme
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
 
 }
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index d80c2ea7a..7d3d52751 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -293,6 +293,6 @@ struct MercurialInputScheme : InputScheme
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
+static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
 
 }
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 99d4b4e8f..bcb904c0d 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -102,6 +102,6 @@ struct PathInputScheme : InputScheme
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
+static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
 
 }
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index ca49482a9..8c0f20475 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -231,6 +231,6 @@ struct TarballInputScheme : InputScheme
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
 
 }
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 49641c2ac..736be8413 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -63,6 +63,6 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
     { unsupported("buildDerivation"); }
 };
 
-static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore;
+static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
 
 }
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index cd619672f..c2c65af05 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -31,7 +31,7 @@ namespace nix {
 
 FileTransferSettings fileTransferSettings;
 
-static GlobalConfig::Register r1(&fileTransferSettings);
+static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
 std::string resolveUri(const std::string & uri)
 {
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index c5734852d..1238dc530 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -25,7 +25,7 @@ namespace nix {
 
 Settings settings;
 
-static GlobalConfig::Register r1(&settings);
+static GlobalConfig::Register rSettings(&settings);
 
 Settings::Settings()
     : nixPrefix(NIX_PREFIX)
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 3d3d91e5e..9d2a89f96 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -181,6 +181,6 @@ protected:
 
 };
 
-static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore;
+static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;
 
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 6db44047d..467169ce8 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -335,6 +335,6 @@ public:
     }
 };
 
-static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore;
+static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
 
 }
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index b5744448e..7d979c5c2 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -105,6 +105,6 @@ std::set<std::string> LocalBinaryCacheStore::uriSchemes()
         return {"file"};
 }
 
-static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore;
+static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regLocalBinaryCacheStore;
 
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 049d4e954..40ef72293 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -1008,6 +1008,6 @@ void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
 
 }
 
-static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
+static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
 
 }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index d43f267e0..552c4aac7 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -439,7 +439,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
 
 };
 
-static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore;
+static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore;
 
 }
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 6d6eca98d..08d0bd565 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -83,6 +83,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
     return conn;
 }
 
-static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore;
+static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
 
 }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 6ad9fa238..f1479329f 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -33,7 +33,7 @@ struct ArchiveSettings : Config
 
 static ArchiveSettings archiveSettings;
 
-static GlobalConfig::Register r1(&archiveSettings);
+static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 
 const std::string narVersionMagic1 = "nix-archive-1";
 
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 147602415..2760b830b 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
     };
 }
 
-static void completePath(std::string_view prefix, bool onlyDirs)
+static void _completePath(std::string_view prefix, bool onlyDirs)
 {
     pathCompletions = true;
     glob_t globbuf;
@@ -300,12 +300,12 @@ static void completePath(std::string_view prefix, bool onlyDirs)
 
 void completePath(size_t, std::string_view prefix)
 {
-    completePath(prefix, false);
+    _completePath(prefix, false);
 }
 
 void completeDir(size_t, std::string_view prefix)
 {
-    completePath(prefix, true);
+    _completePath(prefix, true);
 }
 
 Strings argvToStrings(int argc, char * * argv)
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index cbbf64395..8a6752e22 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -10,7 +10,7 @@ namespace nix {
 
 LoggerSettings loggerSettings;
 
-static GlobalConfig::Register r1(&loggerSettings);
+static GlobalConfig::Register rLoggerSettings(&loggerSettings);
 
 static thread_local ActivityId curActivity = 0;
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 3a8d67f21..a79b1086b 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -67,7 +67,7 @@ std::vector<string> shellwords(const string & s)
     return res;
 }
 
-static void _main(int argc, char * * argv)
+static void main_nix_build(int argc, char * * argv)
 {
     auto dryRun = false;
     auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$"));
@@ -540,5 +540,5 @@ static void _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-build", _main);
-static RegisterLegacyCommand s2("nix-shell", _main);
+static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build);
+static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build);
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index e48f7af9a..309970df6 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -153,7 +153,7 @@ static void update(const StringSet & channelNames)
     replaceSymlink(profile, channelLink);
 }
 
-static int _main(int argc, char ** argv)
+static int main_nix_channel(int argc, char ** argv)
 {
     {
         // Figure out the name of the `.nix-channels' file to use
@@ -250,4 +250,4 @@ static int _main(int argc, char ** argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-channel", _main);
+static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel);
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index bcf1d8518..57092b887 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -49,7 +49,7 @@ void removeOldGenerations(std::string dir)
     }
 }
 
-static int _main(int argc, char * * argv)
+static int main_nix_collect_garbage(int argc, char * * argv)
 {
     {
         bool removeOld = false;
@@ -92,4 +92,4 @@ static int _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-collect-garbage", _main);
+static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage);
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index b10184718..10990f7b5 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -4,7 +4,7 @@
 
 using namespace nix;
 
-static int _main(int argc, char ** argv)
+static int main_nix_copy_closure(int argc, char ** argv)
 {
     {
         auto gzip = false;
@@ -65,4 +65,4 @@ static int _main(int argc, char ** argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-copy-closure", _main);
+static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure);
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 6e652ccbf..fc6195cf0 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -265,7 +265,7 @@ static void daemonLoop(char * * argv)
 }
 
 
-static int _main(int argc, char * * argv)
+static int main_nix_daemon(int argc, char * * argv)
 {
     {
         auto stdio = false;
@@ -330,4 +330,4 @@ static int _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-daemon", _main);
+static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon);
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 3e7c453fb..e6667e7f5 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1330,7 +1330,7 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs)
 }
 
 
-static int _main(int argc, char * * argv)
+static int main_nix_env(int argc, char * * argv)
 {
     {
         Strings opFlags, opArgs;
@@ -1460,4 +1460,4 @@ static int _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-env", _main);
+static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env);
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 539092cbe..18a0049a6 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -83,7 +83,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
 }
 
 
-static int _main(int argc, char * * argv)
+static int main_nix_instantiate(int argc, char * * argv)
 {
     {
         Strings files;
@@ -192,4 +192,4 @@ static int _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-instantiate", _main);
+static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate);
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 377ae03a8..3bdee55a7 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -48,7 +48,7 @@ string resolveMirrorUri(EvalState & state, string uri)
 }
 
 
-static int _main(int argc, char * * argv)
+static int main_nix_prefetch_url(int argc, char * * argv)
 {
     {
         HashType ht = htSHA256;
@@ -229,4 +229,4 @@ static int _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-prefetch-url", _main);
+static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index da91caef1..14baabc36 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -24,6 +24,9 @@
 #endif
 
 
+namespace nix_store {
+
+
 using namespace nix;
 using std::cin;
 using std::cout;
@@ -1025,7 +1028,7 @@ static void opVersion(Strings opFlags, Strings opArgs)
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
-static int _main(int argc, char * * argv)
+static int main_nix_store(int argc, char * * argv)
 {
     {
         Strings opFlags, opArgs;
@@ -1121,4 +1124,6 @@ static int _main(int argc, char * * argv)
     }
 }
 
-static RegisterLegacyCommand s1("nix-store", _main);
+static RegisterLegacyCommand r_nix_store("nix-store", main_nix_store);
+
+}
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 023ffa4ed..7fe87d757 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -87,4 +87,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     }
 };
 
-static auto r1 = registerCommand<CmdAddToStore>("add-to-store");
+static auto rCmdAddToStore = registerCommand<CmdAddToStore>("add-to-store");
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 4605eb13e..d85a482db 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -88,4 +88,4 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
     }
 };
 
-static auto r1 = registerCommand<CmdBuild>("build");
+static auto rCmdBuild = registerCommand<CmdBuild>("build");
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 97306107c..eef172cfc 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -72,5 +72,5 @@ struct CmdCatNar : StoreCommand, MixCat
     }
 };
 
-static auto r1 = registerCommand<CmdCatStore>("cat-store");
-static auto r2 = registerCommand<CmdCatNar>("cat-nar");
+static auto rCmdCatStore = registerCommand<CmdCatStore>("cat-store");
+static auto rCmdCatNar = registerCommand<CmdCatNar>("cat-nar");
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 811c8ab34..cb31aac8f 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -106,4 +106,4 @@ struct CmdCopy : StorePathsCommand
     }
 };
 
-static auto r1 = registerCommand<CmdCopy>("copy");
+static auto rCmdCopy = registerCommand<CmdCopy>("copy");
diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc
index 0cc2d9337..1dd384c0e 100644
--- a/src/nix/describe-stores.cc
+++ b/src/nix/describe-stores.cc
@@ -41,4 +41,4 @@ struct CmdDescribeStores : Command, MixJSON
     }
 };
 
-static auto r1 = registerCommand<CmdDescribeStores>("describe-stores");
+static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores");
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index f29fa71d2..a46ea39b6 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -443,5 +443,5 @@ struct CmdPrintDevEnv : Common
     }
 };
 
-static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
-static auto r2 = registerCommand<CmdDevelop>("develop");
+static auto rCmdPrintDevEnv = registerCommand<CmdPrintDevEnv>("print-dev-env");
+static auto rCmdDevelop = registerCommand<CmdDevelop>("develop");
diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc
index 0dc99d05e..30e7b20e1 100644
--- a/src/nix/diff-closures.cc
+++ b/src/nix/diff-closures.cc
@@ -143,4 +143,4 @@ struct CmdDiffClosures : SourceExprCommand
     }
 };
 
-static auto r1 = registerCommand<CmdDiffClosures>("diff-closures");
+static auto rCmdDiffClosures = registerCommand<CmdDiffClosures>("diff-closures");
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 683e91446..4588ac05e 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -131,4 +131,4 @@ struct CmdDoctor : StoreCommand
     }
 };
 
-static auto r1 = registerCommand<CmdDoctor>("doctor");
+static auto rCmdDoctor = registerCommand<CmdDoctor>("doctor");
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index e1de71bf8..6fd197531 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -30,4 +30,4 @@ struct CmdDumpPath : StorePathCommand
     }
 };
 
-static auto r1 = registerCommand<CmdDumpPath>("dump-path");
+static auto rDumpPath = registerCommand<CmdDumpPath>("dump-path");
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 378a3739c..51c16f5a9 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -54,4 +54,4 @@ struct CmdEdit : InstallableCommand
     }
 };
 
-static auto r1 = registerCommand<CmdEdit>("edit");
+static auto rCmdEdit = registerCommand<CmdEdit>("edit");
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index a8ca446be..43ce46546 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -90,4 +90,4 @@ struct CmdEval : MixJSON, InstallableCommand
     }
 };
 
-static auto r1 = registerCommand<CmdEval>("eval");
+static auto rCmdEval = registerCommand<CmdEval>("eval");
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index ae6f4c5f9..d45f13029 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -965,4 +965,4 @@ struct CmdFlake : NixMultiCommand
     }
 };
 
-static auto r1 = registerCommand<CmdFlake>("flake");
+static auto rCmdFlake = registerCommand<CmdFlake>("flake");
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 494f00a20..1d23bb0e2 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -79,8 +79,8 @@ struct CmdHash : Command
     }
 };
 
-static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
-static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
+static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
+static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
 
 struct CmdToBase : Command
 {
@@ -112,10 +112,10 @@ struct CmdToBase : Command
     }
 };
 
-static RegisterCommand r3("to-base16", [](){ return make_ref<CmdToBase>(Base16); });
-static RegisterCommand r4("to-base32", [](){ return make_ref<CmdToBase>(Base32); });
-static RegisterCommand r5("to-base64", [](){ return make_ref<CmdToBase>(Base64); });
-static RegisterCommand r6("to-sri", [](){ return make_ref<CmdToBase>(SRI); });
+static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref<CmdToBase>(Base16); });
+static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref<CmdToBase>(Base32); });
+static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref<CmdToBase>(Base64); });
+static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref<CmdToBase>(SRI); });
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
@@ -167,4 +167,4 @@ static int compatNixHash(int argc, char * * argv)
     return 0;
 }
 
-static RegisterLegacyCommand s1("nix-hash", compatNixHash);
+static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash);
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 33380dcf5..33a3053f5 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -64,4 +64,4 @@ struct CmdLog : InstallableCommand
     }
 };
 
-static auto r1 = registerCommand<CmdLog>("log");
+static auto rCmdLog = registerCommand<CmdLog>("log");
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 76c8bc9a3..baca54431 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -152,5 +152,5 @@ struct CmdLsNar : Command, MixLs
     }
 };
 
-static auto r1 = registerCommand<CmdLsStore>("ls-store");
-static auto r2 = registerCommand<CmdLsNar>("ls-nar");
+static auto rCmdLsStore = registerCommand<CmdLsStore>("ls-store");
+static auto rCmdLsNar = registerCommand<CmdLsNar>("ls-nar");
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 38b60fc38..df3ec5194 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -108,4 +108,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
     }
 };
 
-static auto r1 = registerCommand<CmdMakeContentAddressable>("make-content-addressable");
+static auto rCmdMakeContentAddressable = registerCommand<CmdMakeContentAddressable>("make-content-addressable");
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index b45951879..51a7a9756 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -31,4 +31,4 @@ struct CmdOptimiseStore : StoreCommand
     }
 };
 
-static auto r1 = registerCommand<CmdOptimiseStore>("optimise-store");
+static auto rCmdOptimiseStore = registerCommand<CmdOptimiseStore>("optimise-store");
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 0c12efaf0..63cf885f9 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -127,4 +127,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
     }
 };
 
-static auto r1 = registerCommand<CmdPathInfo>("path-info");
+static auto rCmdPathInfo = registerCommand<CmdPathInfo>("path-info");
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 127397a29..8db78d591 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -29,4 +29,4 @@ struct CmdPingStore : StoreCommand
     }
 };
 
-static auto r1 = registerCommand<CmdPingStore>("ping-store");
+static auto rCmdPingStore = registerCommand<CmdPingStore>("ping-store");
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 7ce4dfe4c..01aef2f9b 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -467,4 +467,4 @@ struct CmdProfile : NixMultiCommand
     }
 };
 
-static auto r1 = registerCommand<CmdProfile>("profile");
+static auto rCmdProfile = registerCommand<CmdProfile>("profile");
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index cb11ec195..8e8983ad0 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -143,4 +143,4 @@ struct CmdRegistry : virtual NixMultiCommand
     }
 };
 
-static auto r1 = registerCommand<CmdRegistry>("registry");
+static auto rCmdRegistry = registerCommand<CmdRegistry>("registry");
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 329999475..9ff386b1d 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -825,6 +825,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs
     }
 };
 
-static auto r1 = registerCommand<CmdRepl>("repl");
+static auto rCmdRepl = registerCommand<CmdRepl>("repl");
 
 }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index e6584346e..790784382 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -140,7 +140,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
     }
 };
 
-static auto r1 = registerCommand<CmdShell>("shell");
+static auto rCmdShell = registerCommand<CmdShell>("shell");
 
 struct CmdRun : InstallableCommand, RunCommon
 {
@@ -209,7 +209,7 @@ struct CmdRun : InstallableCommand, RunCommon
     }
 };
 
-static auto r2 = registerCommand<CmdRun>("run");
+static auto rCmdRun = registerCommand<CmdRun>("run");
 
 void chrootHelper(int argc, char * * argv)
 {
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 430979274..d4326dc84 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -185,4 +185,4 @@ struct CmdSearch : InstallableCommand, MixJSON
     }
 };
 
-static auto r1 = registerCommand<CmdSearch>("search");
+static auto rCmdSearch = registerCommand<CmdSearch>("search");
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 3ed1ad2aa..1ef54a33a 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -30,4 +30,4 @@ struct CmdShowConfig : Command, MixJSON
     }
 };
 
-static auto r1 = registerCommand<CmdShowConfig>("show-config");
+static auto rShowConfig = registerCommand<CmdShowConfig>("show-config");
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index b9f33499b..2542537d3 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -123,4 +123,4 @@ struct CmdShowDerivation : InstallablesCommand
     }
 };
 
-static auto r1 = registerCommand<CmdShowDerivation>("show-derivation");
+static auto rCmdShowDerivation = registerCommand<CmdShowDerivation>("show-derivation");
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 7821a5432..44916c77f 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -92,7 +92,7 @@ struct CmdCopySigs : StorePathsCommand
     }
 };
 
-static auto r1 = registerCommand<CmdCopySigs>("copy-sigs");
+static auto rCmdCopySigs = registerCommand<CmdCopySigs>("copy-sigs");
 
 struct CmdSignPaths : StorePathsCommand
 {
@@ -144,4 +144,4 @@ struct CmdSignPaths : StorePathsCommand
     }
 };
 
-static auto r2 = registerCommand<CmdSignPaths>("sign-paths");
+static auto rCmdSignPaths = registerCommand<CmdSignPaths>("sign-paths");
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index a880bdae0..66ecc5b34 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -158,4 +158,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
     }
 };
 
-static auto r1 = registerCommand<CmdUpgradeNix>("upgrade-nix");
+static auto rCmdUpgradeNix = registerCommand<CmdUpgradeNix>("upgrade-nix");
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 26f755fd9..ec7333d03 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -189,4 +189,4 @@ struct CmdVerify : StorePathsCommand
     }
 };
 
-static auto r1 = registerCommand<CmdVerify>("verify");
+static auto rCmdVerify = registerCommand<CmdVerify>("verify");
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 7e630b745..63bf087e6 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -263,4 +263,4 @@ struct CmdWhyDepends : SourceExprCommand
     }
 };
 
-static auto r1 = registerCommand<CmdWhyDepends>("why-depends");
+static auto rCmdWhyDepends = registerCommand<CmdWhyDepends>("why-depends");

From ad143c5b3b7e713b89f0437ce17c20ac642ca530 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 14:00:42 +0200
Subject: [PATCH 331/882] Shut up some clang warnings

---
 src/libexpr/eval-cache.hh   | 2 +-
 src/libfetchers/fetchers.hh | 3 +++
 src/libfetchers/github.cc   | 6 +++---
 src/libstore/dummy-store.cc | 3 +--
 src/libstore/store-api.hh   | 2 ++
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index 8ffffc0ed..e23e45c94 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -11,7 +11,7 @@ namespace nix::eval_cache {
 
 MakeError(CachedEvalError, EvalError);
 
-class AttrDb;
+struct AttrDb;
 class AttrCursor;
 
 class EvalCache : public std::enable_shared_from_this<EvalCache>
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index cc31a31b9..e8ae59143 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -86,6 +86,9 @@ public:
 
 struct InputScheme
 {
+    virtual ~InputScheme()
+    { }
+
     virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
 
     virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 90893075e..92ff224f7 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -25,7 +25,7 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::string type() = 0;
 
-    virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
+    virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
@@ -215,7 +215,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const
+    std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Github supports PAT/OAuth2 tokens and HTTP Basic
         // Authentication.  The former simply specifies the token, the
@@ -270,7 +270,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
-    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const
+    std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Gitlab supports 4 kinds of authorization, two of which are
         // relevant here: OAuth2 and PAT (Private Access Token).  The
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 736be8413..98b745c3a 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -18,8 +18,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
     DummyStore(const Params & params)
         : StoreConfig(params)
         , Store(params)
-    {
-    }
+    { }
 
     string getUri() override
     {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 854446987..450c0f554 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -194,6 +194,8 @@ struct StoreConfig : public Config
      */
     StoreConfig() { assert(false); }
 
+    virtual ~StoreConfig() { }
+
     virtual const std::string name() = 0;
 
     const PathSetting storeDir_{this, false, settings.nixStore,

From d761485010cc9d071da17a5d90cf4126a4bf499d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Oct 2020 18:52:46 +0200
Subject: [PATCH 332/882] Prevent a deadlock when user namespace setup fails

Observed on Centos 7 when user namespaces are disabled:
DerivationGoal::startBuilder() throws an exception, ~DerivationGoal()
waits for the child process to exit, but the child process hangs
forever in drainFD(userNamespaceSync.readSide.get()) in
DerivationGoal::runChild(). Not sure why the SIGKILL doesn't get
through.

Issue #4092.
---
 src/libstore/build.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 87e01a378..9497e7b3e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2686,6 +2686,12 @@ void DerivationGoal::startBuilder()
 
         userNamespaceSync.readSide = -1;
 
+        /* Close the write side to prevent runChild() from hanging
+           reading from this. */
+        Finally cleanup([&]() {
+            userNamespaceSync.writeSide = -1;
+        });
+
         pid_t tmp;
         if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort();
         pid = tmp;
@@ -2712,7 +2718,6 @@ void DerivationGoal::startBuilder()
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
-        userNamespaceSync.writeSide = -1;
 
     } else
 #endif

From 59f2dd8e8da1f82aa9e29e30ba1df643434a9254 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 6 Oct 2020 20:08:51 +0200
Subject: [PATCH 333/882] libfetchers/github: allow slashes in refs

Refs #4061
---
 src/libfetchers/github.cc | 20 +++++++++++++++++---
 src/libutil/url-parts.hh  |  2 +-
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 92ff224f7..3d1cc15e2 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme
         std::optional<std::string> ref;
         std::optional<std::string> host_url;
 
-        if (path.size() == 2) {
-        } else if (path.size() == 3) {
+        auto size = path.size();
+        if (size == 3) {
             if (std::regex_match(path[2], revRegex))
                 rev = Hash::parseAny(path[2], htSHA1);
             else if (std::regex_match(path[2], refRegex))
                 ref = path[2];
             else
                 throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
-        } else
+        } else if (size > 3) {
+            std::string rs;
+            for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
+                rs += *i;
+                if (std::next(i) != path.end()) {
+                    rs += "/";
+                }
+            }
+
+            if (std::regex_match(rs, refRegex)) {
+                ref = rs;
+            } else {
+                throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs);
+            }
+        } else if (size < 2)
             throw BadURL("URL '%s' is invalid", url.url);
 
         for (auto &[name, value] : url.query) {
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 68be15cb0..e0e2809fd 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -22,7 +22,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
 const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
 
 // A Git ref (i.e. branch or tag name).
-const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check
 extern std::regex refRegex;
 
 // Instead of defining what a good Git Ref is, we define what a bad Git Ref is

From 57d960dcd174f0910c7fca02ee3afe80cb2b7844 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 7 Oct 2020 12:48:35 +0000
Subject: [PATCH 334/882] Remove generic std::optional<T> suppport from worker
 proto

See comment for rational; I think it's good to leave a comment lest
anyone is tempted to add such a sum-type instance again.

Fixes #4113
---
 src/libstore/worker-protocol.hh | 48 +++++++++++----------------------
 1 file changed, 16 insertions(+), 32 deletions(-)

diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 2934c1d67..564fb978a 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -83,7 +83,6 @@ MAKE_WORKER_PROTO(, StorePath);
 MAKE_WORKER_PROTO(, ContentAddress);
 
 MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
-MAKE_WORKER_PROTO(template<typename T>, std::optional<T>);
 
 #define X_ template<typename K, typename V>
 #define Y_ std::map<K, V>
@@ -91,6 +90,22 @@ MAKE_WORKER_PROTO(X_, Y_);
 #undef X_
 #undef Y_
 
+/* These use the empty string for the null case, relying on the fact
+   that the underlying types never serialize to the empty string.
+
+   We do this instead of a generic std::optional<T> instance because
+   ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
+   the same reason, we don't have a std::variant<T..> instances (ordinal
+   tags 0...n).
+
+   We could the generic instances and then these as specializations for
+   compatability, but that's proven a bit finnicky, and also makes the
+   worker protocol harder to implement in other languages where such
+   specializations may not be allowed.
+ */
+MAKE_WORKER_PROTO(, std::optional<StorePath>);
+MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
+
 template<typename T>
 std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
 {
@@ -134,37 +149,6 @@ void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
     }
 }
 
-template<typename T>
-std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _)
-{
-    auto tag = readNum<uint8_t>(from);
-    switch (tag) {
-    case 0:
-        return std::nullopt;
-    case 1:
-        return read(store, from, Phantom<T> {});
-    default:
-        throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
-    }
-}
-
-template<typename T>
-void write(const Store & store, Sink & out, const std::optional<T> & optVal)
-{
-    out << (uint64_t) (optVal ? 1 : 0);
-    if (optVal)
-        worker_proto::write(store, out, *optVal);
-}
-
-/* Specialization which uses and empty string for the empty case, taking
-   advantage of the fact these types always serialize to non-empty strings.
-   This is done primarily for backwards compatability, so that T <=
-   std::optional<T>, where <= is the compatability partial order, T is one of
-   the types below.
- */
-MAKE_WORKER_PROTO(, std::optional<StorePath>);
-MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
-
 }
 
 }

From 27ca87c46a57e1e6603028c1902189b53635d576 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 16:33:19 +0200
Subject: [PATCH 335/882] Formatting

---
 src/libutil/error.cc | 26 +++++++++++++-------------
 src/libutil/error.hh |  4 ++--
 src/libutil/fmt.hh   | 30 +++++++++++++++++-------------
 3 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index fd6f69b7f..803a72953 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -11,13 +11,13 @@ const std::string nativeSystem = SYSTEM;
 
 BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
 {
-    err.traces.push_front(Trace { .pos = e, .hint = hint});
+    err.traces.push_front(Trace { .pos = e, .hint = hint });
     return *this;
 }
 
 // c++ std::exception descendants must have a 'const char* what()' function.
 // This stringifies the error and caches it for use by what(), or similarly by msg().
-const string& BaseError::calcWhat() const
+const string & BaseError::calcWhat() const
 {
     if (what_.has_value())
         return *what_;
@@ -34,12 +34,12 @@ const string& BaseError::calcWhat() const
 
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
-std::ostream& operator<<(std::ostream &os, const hintformat &hf)
+std::ostream & operator<<(std::ostream & os, const hintformat & hf)
 {
     return os << hf.str();
 }
 
-string showErrPos(const ErrPos &errPos)
+string showErrPos(const ErrPos & errPos)
 {
     if (errPos.line > 0) {
         if (errPos.column > 0) {
@@ -53,7 +53,7 @@ string showErrPos(const ErrPos &errPos)
     }
 }
 
-std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
+std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
 {
     if (errPos.line <= 0)
         return std::nullopt;
@@ -92,13 +92,13 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
                 return loc;
             }
         }
-        catch (EndOfFile &eof) {
+        catch (EndOfFile & eof) {
             if (loc.errLineOfCode.has_value())
                 return loc;
             else
                 return std::nullopt;
         }
-        catch (std::exception &e) {
+        catch (std::exception & e) {
             printError("error reading nix file: %s\n%s", errPos.file, e.what());
             return std::nullopt;
         }
@@ -137,10 +137,10 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
 }
 
 // print lines of code to the ostream, indicating the error column.
-void printCodeLines(std::ostream &out,
-    const string &prefix,
-    const ErrPos &errPos,
-    const LinesOfCode &loc)
+void printCodeLines(std::ostream & out,
+    const string & prefix,
+    const ErrPos & errPos,
+    const LinesOfCode & loc)
 {
     // previous line of code.
     if (loc.prevLineOfCode.has_value()) {
@@ -186,7 +186,7 @@ void printCodeLines(std::ostream &out,
     }
 }
 
-void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out)
+void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
 {
     if (pos)
     {
@@ -212,7 +212,7 @@ void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out)
     }
 }
 
-std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace)
+std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
 {
     auto errwidth = std::max<size_t>(getWindowSize().second, 20);
     string prefix = "";
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index f3babcbde..d1b6d82bb 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -107,7 +107,7 @@ struct Trace {
 struct ErrorInfo {
     Verbosity level;
     string name;
-    string description;
+    string description; // FIXME: remove? it seems to be barely used
     std::optional<hintformat> hint;
     std::optional<ErrPos> errPos;
     std::list<Trace> traces;
@@ -169,7 +169,7 @@ public:
 #endif
 
     const string & msg() const { return calcWhat(); }
-    const ErrorInfo & info() { calcWhat(); return err; }
+    const ErrorInfo & info() const { calcWhat(); return err; }
 
     template<typename... Args>
     BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index 6e69bdce2..85c0e9429 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -76,11 +76,11 @@ template <class T>
 struct yellowtxt
 {
     yellowtxt(const T &s) : value(s) {}
-    const T &value;
+    const T & value;
 };
 
 template <class T>
-std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y)
+std::ostream & operator<<(std::ostream & out, const yellowtxt<T> & y)
 {
     return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
 }
@@ -88,12 +88,12 @@ std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y)
 template <class T>
 struct normaltxt
 {
-    normaltxt(const T &s) : value(s) {}
-    const T &value;
+    normaltxt(const T & s) : value(s) {}
+    const T & value;
 };
 
 template <class T>
-std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y)
+std::ostream & operator<<(std::ostream & out, const normaltxt<T> & y)
 {
     return out << ANSI_NORMAL << y.value;
 }
@@ -101,26 +101,30 @@ std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y)
 class hintformat
 {
 public:
-    hintformat(const string &format) :fmt(format)
+    hintformat(const string & format) : fmt(format)
     {
-        fmt.exceptions(boost::io::all_error_bits ^ 
+        fmt.exceptions(boost::io::all_error_bits ^
                        boost::io::too_many_args_bit ^
                        boost::io::too_few_args_bit);
     }
 
-    hintformat(const hintformat &hf)
-    : fmt(hf.fmt)
-    {}
+    hintformat(const hintformat & hf)
+        : fmt(hf.fmt)
+    { }
+
+    hintformat(format && fmt)
+        : fmt(std::move(fmt))
+    { }
 
     template<class T>
-    hintformat& operator%(const T &value)
+    hintformat & operator%(const T & value)
     {
         fmt % yellowtxt(value);
         return *this;
     }
 
     template<class T>
-    hintformat& operator%(const normaltxt<T> &value)
+    hintformat & operator%(const normaltxt<T> & value)
     {
         fmt % value.value;
         return *this;
@@ -135,7 +139,7 @@ private:
     format fmt;
 };
 
-std::ostream& operator<<(std::ostream &os, const hintformat &hf);
+std::ostream & operator<<(std::ostream & os, const hintformat & hf);
 
 template<typename... Args>
 inline hintformat hintfmt(const std::string & fs, const Args & ... args)

From be149acfdaae06667bc58df467c04c41827b69d0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 16:34:03 +0200
Subject: [PATCH 336/882] Serialize exceptions from the sandbox process to the
 parent

Fixes #4118.
---
 src/libstore/build.cc    | 16 ++++++++++-----
 src/libutil/serialise.cc | 42 ++++++++++++++++++++++++++++++++++++++++
 src/libutil/serialise.hh |  3 +++
 3 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9497e7b3e..fa7283588 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2737,9 +2737,12 @@ void DerivationGoal::startBuilder()
     /* Check if setting up the build environment failed. */
     while (true) {
         string msg = readLine(builderOut.readSide.get());
+        if (string(msg, 0, 1) == "\2") break;
         if (string(msg, 0, 1) == "\1") {
-            if (msg.size() == 1) break;
-            throw Error(string(msg, 1));
+            FdSource source(builderOut.readSide.get());
+            auto ex = readError(source);
+            ex.addTrace({}, "while setting up the build environment");
+            throw ex;
         }
         debug(msg);
     }
@@ -3785,7 +3788,7 @@ void DerivationGoal::runChild()
             args.push_back(rewriteStrings(i, inputRewrites));
 
         /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\1\n"));
+        writeFull(STDERR_FILENO, string("\2\n"));
 
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
@@ -3815,8 +3818,11 @@ void DerivationGoal::runChild()
 
         throw SysError("executing '%1%'", drv->builder);
 
-    } catch (std::exception & e) {
-        writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n");
+    } catch (Error & e) {
+        writeFull(STDERR_FILENO, "\1\n");
+        FdSink sink(STDERR_FILENO);
+        sink << e;
+        sink.flush();
         _exit(1);
     }
 }
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index a469a1e73..5c9f6f901 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -266,6 +266,24 @@ Sink & operator << (Sink & sink, const StringSet & s)
     return sink;
 }
 
+Sink & operator << (Sink & sink, const Error & ex)
+{
+    auto info = ex.info();
+    sink
+        << "Error"
+        << info.level
+        << info.name
+        << info.description
+        << (info.hint ? info.hint->str() : "")
+        << 0 // FIXME: info.errPos
+        << info.traces.size();
+    for (auto & trace : info.traces) {
+        sink << 0; // FIXME: trace.pos
+        sink << trace.hint.str();
+    }
+    return sink;
+}
+
 
 void readPadding(size_t len, Source & source)
 {
@@ -319,6 +337,30 @@ template Paths readStrings(Source & source);
 template PathSet readStrings(Source & source);
 
 
+Error readError(Source & source)
+{
+    auto type = readString(source);
+    assert(type == "Error");
+    ErrorInfo info;
+    info.level = (Verbosity) readInt(source);
+    info.name = readString(source);
+    info.description = readString(source);
+    auto hint = readString(source);
+    if (hint != "") info.hint = hintformat(std::move(format("%s") % hint));
+    auto havePos = readNum<size_t>(source);
+    assert(havePos == 0);
+    auto nrTraces = readNum<size_t>(source);
+    for (size_t i = 0; i < nrTraces; ++i) {
+        havePos = readNum<size_t>(source);
+        assert(havePos == 0);
+        info.traces.push_back(Trace {
+            .hint = hintformat(std::move(format("%s") % readString(source)))
+        });
+    }
+    return Error(std::move(info));
+}
+
+
 void StringSink::operator () (const unsigned char * data, size_t len)
 {
     static bool warned = false;
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index b41e58f33..d7fe0b81e 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -321,6 +321,7 @@ inline Sink & operator << (Sink & sink, uint64_t n)
 Sink & operator << (Sink & sink, const string & s);
 Sink & operator << (Sink & sink, const Strings & s);
 Sink & operator << (Sink & sink, const StringSet & s);
+Sink & operator << (Sink & in, const Error & ex);
 
 
 MakeError(SerialisationError, Error);
@@ -382,6 +383,8 @@ Source & operator >> (Source & in, bool & b)
     return in;
 }
 
+Error readError(Source & source);
+
 
 /* An adapter that converts a std::basic_istream into a source. */
 struct StreamToSourceAdapter : Source

From c43e882f54b5f09278dfb0d315239531c0676b4e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 17:13:54 +0200
Subject: [PATCH 337/882] Serialize exceptions from the daemon to the client

---
 src/libstore/daemon.cc          | 23 ++++++++++++++++-------
 src/libstore/remote-store.cc    | 10 +++++++---
 src/libstore/worker-protocol.hh |  2 +-
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index ae2fbec35..99d8add92 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -101,17 +101,20 @@ struct TunnelLogger : public Logger
 
     /* stopWork() means that we're done; stop sending stderr to the
        client. */
-    void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
+    void stopWork(const Error * ex = nullptr)
     {
         auto state(state_.lock());
 
         state->canSendStderr = false;
 
-        if (success)
+        if (!ex)
             to << STDERR_LAST;
         else {
-            to << STDERR_ERROR << msg;
-            if (status != 0) to << status;
+            if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
+                to << STDERR_ERROR << *ex;
+            } else {
+                to << STDERR_ERROR << ex->what() << ex->status;
+            }
         }
     }
 
@@ -935,10 +938,11 @@ void processConnection(
                    during addTextToStore() / importPath().  If that
                    happens, just send the error message and exit. */
                 bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
-                tunnelLogger->stopWork(false, e.msg(), e.status);
+                tunnelLogger->stopWork(&e);
                 if (!errorAllowed) throw;
             } catch (std::bad_alloc & e) {
-                tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
+                auto ex = Error("Nix daemon out of memory");
+                tunnelLogger->stopWork(&ex);
                 throw;
             }
 
@@ -947,8 +951,13 @@ void processConnection(
             assert(!tunnelLogger->state_.lock()->canSendStderr);
         };
 
+    } catch (Error & e) {
+        tunnelLogger->stopWork(&e);
+        to.flush();
+        return;
     } catch (std::exception & e) {
-        tunnelLogger->stopWork(false, e.what(), 1);
+        auto ex = Error(e.what());
+        tunnelLogger->stopWork(&ex);
         to.flush();
         return;
     }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 40ef72293..23b1942ce 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -925,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         }
 
         else if (msg == STDERR_ERROR) {
-            string error = readString(from);
-            unsigned int status = readInt(from);
-            return std::make_exception_ptr(Error(status, error));
+            if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
+                return std::make_exception_ptr(readError(from));
+            } else {
+                string error = readString(from);
+                unsigned int status = readInt(from);
+                return std::make_exception_ptr(Error(status, error));
+            }
         }
 
         else if (msg == STDERR_NEXT)
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 564fb978a..b3705578e 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x119
+#define PROTOCOL_VERSION 0x11a
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

From e705c2429445a4a05076fc067bb349bdd6752bd9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 17:28:43 +0200
Subject: [PATCH 338/882] Tweak error messages

---
 src/libstore/build.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index fa7283588..0585905e0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2355,7 +2355,8 @@ void DerivationGoal::startBuilder()
                     worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
-                throw Error("while processing 'sandbox-paths': %s", e.what());
+                e.addTrace({}, "while processing 'sandbox-paths'");
+                throw;
             }
         for (auto & i : closure) {
             auto p = worker.store.printStorePath(i);
@@ -3809,7 +3810,7 @@ void DerivationGoal::runChild()
                     throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
                 _exit(0);
             } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n");
+                writeFull(STDERR_FILENO, e.what() + "\n");
                 _exit(1);
             }
         }

From f66bbd8c7bb1472facf8917e58e3cd4f6ddfa1b5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 21:25:06 +0200
Subject: [PATCH 339/882] Doh

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0585905e0..e1774237b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3810,7 +3810,7 @@ void DerivationGoal::runChild()
                     throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
                 _exit(0);
             } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + "\n");
+                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
                 _exit(1);
             }
         }

From 6aa64627c8e431c3b187f7bb44c943d06e39b929 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 22:02:36 +0200
Subject: [PATCH 340/882] Support user namespaces being disabled

If max_user_namespaces is set to 0, then don't run the build in a user
namespace.

Fixes #4092.
---
 src/libstore/build.cc | 54 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 14 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e1774237b..a645c429d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -831,6 +831,10 @@ private:
        paths to the sandbox as a result of recursive Nix calls. */
     AutoCloseFD sandboxMountNamespace;
 
+    /* On Linux, whether we're doing the build in its own user
+       namespace. */
+    bool usingUserNamespace = true;
+
     /* The build hook. */
     std::unique_ptr<HookInstance> hook;
 
@@ -920,8 +924,8 @@ private:
        result. */
     std::map<Path, ValidPathInfo> prevInfos;
 
-    const uid_t sandboxUid = 1000;
-    const gid_t sandboxGid = 100;
+    uid_t sandboxUid = 1000;
+    gid_t sandboxGid = 100;
 
     const static Path homeDir;
 
@@ -2629,6 +2633,24 @@ void DerivationGoal::startBuilder()
 
         options.allowVfork = false;
 
+        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
+        static bool userNamespacesEnabled =
+            pathExists(maxUserNamespaces)
+            && trim(readFile(maxUserNamespaces)) != "0";
+
+        usingUserNamespace = userNamespacesEnabled;
+
+        if (usingUserNamespace) {
+            sandboxUid = 1000;
+            sandboxGid = 100;
+        } else {
+            debug("note: not using a user namespace");
+            if (!buildUser)
+                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
+            sandboxUid = buildUser->getUID();
+            sandboxGid = buildUser->getGID();
+        }
+
         Pid helper = startProcess([&]() {
 
             /* Drop additional groups here because we can't do it
@@ -2647,9 +2669,11 @@ void DerivationGoal::startBuilder()
                 PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
             if (stack == MAP_FAILED) throw SysError("allocating stack");
 
-            int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
+            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
             if (privateNetwork)
                 flags |= CLONE_NEWNET;
+            if (usingUserNamespace)
+                flags |= CLONE_NEWUSER;
 
             pid_t child = clone(childEntry, stack + stackSize, flags, this);
             if (child == -1 && errno == EINVAL) {
@@ -2697,19 +2721,21 @@ void DerivationGoal::startBuilder()
         if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort();
         pid = tmp;
 
-        /* Set the UID/GID mapping of the builder's user namespace
-           such that the sandbox user maps to the build user, or to
-           the calling user (if build users are disabled). */
-        uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-        uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
+        if (usingUserNamespace) {
+            /* Set the UID/GID mapping of the builder's user namespace
+               such that the sandbox user maps to the build user, or to
+               the calling user (if build users are disabled). */
+            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
+            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
 
-        writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-            (format("%d %d 1") % sandboxUid % hostUid).str());
+            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
+                (format("%d %d 1") % sandboxUid % hostUid).str());
 
-        writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
-        writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-            (format("%d %d 1") % sandboxGid % hostGid).str());
+            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
+                (format("%d %d 1") % sandboxGid % hostGid).str());
+        }
 
         /* Save the mount namespace of the child. We have to do this
            *before* the child does a chroot. */
@@ -2745,7 +2771,7 @@ void DerivationGoal::startBuilder()
             ex.addTrace({}, "while setting up the build environment");
             throw ex;
         }
-        debug(msg);
+        debug("sandbox setup: " + msg);
     }
 }
 

From 97ffc1e0139e124b7e36b5d1a62b90300f231118 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Oct 2020 22:46:01 +0200
Subject: [PATCH 341/882] Dynamically disable user namespaces if CLONE_NEWUSER
 fails

This makes builds work inside nixos-enter.

Fixes #3145.
---
 src/libstore/build.cc | 45 ++++++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a645c429d..05a9ef088 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -924,8 +924,8 @@ private:
        result. */
     std::map<Path, ValidPathInfo> prevInfos;
 
-    uid_t sandboxUid = 1000;
-    gid_t sandboxGid = 100;
+    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
 
     const static Path homeDir;
 
@@ -2428,15 +2428,14 @@ void DerivationGoal::startBuilder()
                 "root:x:0:0:Nix build user:%3%:/noshell\n"
                 "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
                 "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid, sandboxGid, settings.sandboxBuildDir));
+                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
 
         /* Declare the build user's group so that programs get a consistent
            view of the system (e.g., "id -gn"). */
         writeFile(chrootRootDir + "/etc/group",
-            (format(
-                "root:x:0:\n"
+            fmt("root:x:0:\n"
                 "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n") % sandboxGid).str());
+                "nogroup:x:65534:\n", sandboxGid()));
 
         /* Create /etc/hosts with localhost entry. */
         if (!(derivationIsImpure(derivationType)))
@@ -2640,17 +2639,6 @@ void DerivationGoal::startBuilder()
 
         usingUserNamespace = userNamespacesEnabled;
 
-        if (usingUserNamespace) {
-            sandboxUid = 1000;
-            sandboxGid = 100;
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-            sandboxUid = buildUser->getUID();
-            sandboxGid = buildUser->getGID();
-        }
-
         Pid helper = startProcess([&]() {
 
             /* Drop additional groups here because we can't do it
@@ -2682,11 +2670,12 @@ void DerivationGoal::startBuilder()
                 flags &= ~CLONE_NEWPID;
                 child = clone(childEntry, stack + stackSize, flags, this);
             }
-            if (child == -1 && (errno == EPERM || errno == EINVAL)) {
+            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
                 /* Some distros patch Linux to not allow unprivileged
                  * user namespaces. If we get EPERM or EINVAL, try
                  * without CLONE_NEWUSER and see if that works.
                  */
+                usingUserNamespace = false;
                 flags &= ~CLONE_NEWUSER;
                 child = clone(childEntry, stack + stackSize, flags, this);
             }
@@ -2697,7 +2686,8 @@ void DerivationGoal::startBuilder()
                 _exit(1);
             if (child == -1) throw SysError("cloning builder process");
 
-            writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n");
+            writeFull(builderOut.writeSide.get(),
+                fmt("%d %d\n", usingUserNamespace, child));
             _exit(0);
         }, options);
 
@@ -2718,7 +2708,10 @@ void DerivationGoal::startBuilder()
         });
 
         pid_t tmp;
-        if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort();
+        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
+        assert(ss.size() == 2);
+        usingUserNamespace = ss[0] == "1";
+        if (!string2Int<pid_t>(ss[1], tmp)) abort();
         pid = tmp;
 
         if (usingUserNamespace) {
@@ -2729,12 +2722,16 @@ void DerivationGoal::startBuilder()
             uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
 
             writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                (format("%d %d 1") % sandboxUid % hostUid).str());
+                fmt("%d %d 1", sandboxUid(), hostUid));
 
             writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
             writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                (format("%d %d 1") % sandboxGid % hostGid).str());
+                fmt("%d %d 1", sandboxGid(), hostGid));
+        } else {
+            debug("note: not using a user namespace");
+            if (!buildUser)
+                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
         }
 
         /* Save the mount namespace of the child. We have to do this
@@ -3595,9 +3592,9 @@ void DerivationGoal::runChild()
             /* Switch to the sandbox uid/gid in the user namespace,
                which corresponds to the build user or calling user in
                the parent namespace. */
-            if (setgid(sandboxGid) == -1)
+            if (setgid(sandboxGid()) == -1)
                 throw SysError("setgid failed");
-            if (setuid(sandboxUid) == -1)
+            if (setuid(sandboxUid()) == -1)
                 throw SysError("setuid failed");
 
             setUser = false;

From eaef251b2ba416fcde8f0f0b72ce68c548ce35c3 Mon Sep 17 00:00:00 2001
From: Horki <ivanzvonimirhorvat@gmail.com>
Date: Thu, 8 Oct 2020 13:40:47 +0200
Subject: [PATCH 342/882] rust: small patches

---
 nix-rust/src/lib.rs         |  1 +
 nix-rust/src/store/path.rs  | 35 +++++++++++++++++------------------
 nix-rust/src/util/base32.rs |  2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 27ea69fbd..101de106f 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,3 +1,4 @@
+#[allow(improper_ctypes_definitions)]
 #[cfg(not(test))]
 mod c;
 mod error;
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
index 47b5975c0..99f7a1f18 100644
--- a/nix-rust/src/store/path.rs
+++ b/nix-rust/src/store/path.rs
@@ -19,9 +19,9 @@ impl StorePath {
         }
         Self::new_from_base_name(
             path.file_name()
-                .ok_or(Error::BadStorePath(path.into()))?
+                .ok_or_else(|| Error::BadStorePath(path.into()))?
                 .to_str()
-                .ok_or(Error::BadStorePath(path.into()))?,
+                .ok_or_else(|| Error::BadStorePath(path.into()))?,
         )
     }
 
@@ -34,7 +34,7 @@ impl StorePath {
 
     pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
         if base_name.len() < STORE_PATH_HASH_CHARS + 1
-            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
+            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != b'-'
         {
             return Err(Error::BadStorePath(base_name.into()));
         }
@@ -65,7 +65,7 @@ impl StorePathHash {
         Ok(Self(bytes))
     }
 
-    pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] {
+    pub fn hash(&self) -> &[u8; STORE_PATH_HASH_BYTES] {
         &self.0
     }
 }
@@ -98,7 +98,7 @@ pub struct StorePathName(String);
 
 impl StorePathName {
     pub fn new(s: &str) -> Result<Self, Error> {
-        if s.len() == 0 {
+        if s.is_empty() {
             return Err(Error::StorePathNameEmpty);
         }
 
@@ -106,25 +106,24 @@ impl StorePathName {
             return Err(Error::StorePathNameTooLong);
         }
 
-        if s.starts_with('.')
-            || !s.chars().all(|c| {
-                c.is_ascii_alphabetic()
-                    || c.is_ascii_digit()
-                    || c == '+'
-                    || c == '-'
-                    || c == '.'
-                    || c == '_'
-                    || c == '?'
-                    || c == '='
-            })
-        {
+        let is_good_path_name = s.chars().all(|c| {
+            c.is_ascii_alphabetic()
+                || c.is_ascii_digit()
+                || c == '+'
+                || c == '-'
+                || c == '.'
+                || c == '_'
+                || c == '?'
+                || c == '='
+        });
+        if s.starts_with('.') || !is_good_path_name {
             return Err(Error::BadStorePathName);
         }
 
         Ok(Self(s.to_string()))
     }
 
-    pub fn name<'a>(&'a self) -> &'a str {
+    pub fn name(&self) -> &str {
         &self.0
     }
 }
diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs
index efd4a2901..7e71dc920 100644
--- a/nix-rust/src/util/base32.rs
+++ b/nix-rust/src/util/base32.rs
@@ -13,7 +13,7 @@ pub fn decoded_len(input_len: usize) -> usize {
     input_len * 5 / 8
 }
 
-static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
+static BASE32_CHARS: &[u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
 
 lazy_static! {
     static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = {

From 58dadf295499588e492dab6bdc5934dc27ad3e64 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Oct 2020 17:30:40 +0200
Subject: [PATCH 343/882] Remove stray 'Title:' from the manual

Closes #4096.
---
 doc/manual/generate-manpage.nix                   |  3 +--
 doc/manual/local.mk                               | 15 ++++++++++++---
 doc/manual/src/command-ref/conf-file-prefix.md    |  2 --
 doc/manual/src/command-ref/nix-build.md           |  2 --
 doc/manual/src/command-ref/nix-channel.md         |  2 --
 doc/manual/src/command-ref/nix-collect-garbage.md |  2 --
 doc/manual/src/command-ref/nix-copy-closure.md    |  2 --
 doc/manual/src/command-ref/nix-daemon.md          |  2 --
 doc/manual/src/command-ref/nix-env.md             |  2 --
 doc/manual/src/command-ref/nix-hash.md            |  2 --
 doc/manual/src/command-ref/nix-instantiate.md     |  2 --
 doc/manual/src/command-ref/nix-prefetch-url.md    |  2 --
 doc/manual/src/command-ref/nix-shell.md           |  2 --
 doc/manual/src/command-ref/nix-store.md           |  2 --
 14 files changed, 13 insertions(+), 29 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 4709c0e2c..db266750a 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -52,5 +52,4 @@ in
 
 command:
 
-"Title: nix\n\n"
-+ showCommand { command = "nix"; section = "#"; def = command; }
+showCommand { command = "nix"; section = "#"; def = command; }
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index d308c7cf6..7d9a1a3e8 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -18,13 +18,22 @@ dist-files += $(man-pages)
 nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
 
 $(d)/%.1: $(d)/src/command-ref/%.md
-	$(trace-gen) lowdown -sT man $^ -o $@
+	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
+	@cat $^ >> $^.tmp
+	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	@rm $^.tmp
 
 $(d)/%.8: $(d)/src/command-ref/%.md
-	$(trace-gen) lowdown -sT man $^ -o $@
+	@printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp
+	@cat $^ >> $^.tmp
+	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	@rm $^.tmp
 
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
-	$(trace-gen) lowdown -sT man $^ -o $@
+	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
+	@cat $^ >> $^.tmp
+	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	@rm $^.tmp
 
 $(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	$(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp
diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index 04c6cd859..9987393d2 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -1,5 +1,3 @@
-Title: nix.conf
-
 # Name
 
 `nix.conf` - Nix configuration file
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 4bcb8db40..4565bfbc2 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -1,5 +1,3 @@
-Title: nix-build
-
 # Name
 
 `nix-build` - build a Nix expression
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index f0e205967..4ca12d2cc 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -1,5 +1,3 @@
-Title: nix-channel
-
 # Name
 
 `nix-channel` - manage Nix channels
diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index 62a6b7ca0..296165993 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -1,5 +1,3 @@
-Title: nix-collect-garbage
-
 # Name
 
 `nix-collect-garbage` - delete unreachable store paths
diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index 5ce320af7..dcb844a72 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -1,5 +1,3 @@
-Title: nix-copy-closure
-
 # Name
 
 `nix-copy-closure` - copy a closure to or from a remote machine via SSH
diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/src/command-ref/nix-daemon.md
index bd5d25026..e91cb01dd 100644
--- a/doc/manual/src/command-ref/nix-daemon.md
+++ b/doc/manual/src/command-ref/nix-daemon.md
@@ -1,5 +1,3 @@
-Title: nix-daemon
-
 # Name
 
 `nix-daemon` - Nix multi-user support daemon
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index ee838581b..1c23bb0ad 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -1,5 +1,3 @@
-Title: nix-env
-
 # Name
 
 `nix-env` - manipulate or query Nix user environments
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index d3f91f8e9..7ed82cdfc 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -1,5 +1,3 @@
-Title: nix-hash
-
 # Name
 
 `nix-hash` - compute the cryptographic hash of a path
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index d09f5ed6a..c369397b6 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -1,5 +1,3 @@
-Title: nix-instantiate
-
 # Name
 
 `nix-instantiate` - instantiate store derivations from Nix expressions
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index 1307c7c37..78c612cd4 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -1,5 +1,3 @@
-Title: nix-prefetch-url
-
 # Name
 
 `nix-prefetch-url` - copy a file from a URL into the store and print its hash
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index fc42a202a..45a5ff08c 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -1,5 +1,3 @@
-Title: nix-shell
-
 # Name
 
 `nix-shell` - start an interactive shell based on a Nix expression
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 4680339e4..827adbd05 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -1,5 +1,3 @@
-Title: nix-store
-
 # Name
 
 `nix-store` - manipulate or query the Nix store

From 04e5d0e7040fdfbbc084634c0694ae7da89765d9 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 9 Oct 2020 09:39:51 +0200
Subject: [PATCH 344/882] Add a description in the completion outputs

Make nix output completions in the form `completion\tdescription`.
This can't be used by bash (afaik), but other shells like zsh or fish
can display it along the completion choices
---
 misc/bash/completion.sh    |  5 +++--
 src/libmain/common-args.cc |  2 +-
 src/libutil/args.cc        | 29 ++++++++++++++++++++---------
 src/libutil/args.hh        | 12 +++++++++++-
 src/nix/installables.cc    | 12 ++++++------
 src/nix/main.cc            |  2 +-
 6 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh
index bc184edd6..bea2a40bc 100644
--- a/misc/bash/completion.sh
+++ b/misc/bash/completion.sh
@@ -4,13 +4,14 @@ function _complete_nix {
     _get_comp_words_by_ref -n ':=&' words cword cur
     local have_type
     while IFS= read -r line; do
+        local completion=${line%%	*}
         if [[ -z $have_type ]]; then
             have_type=1
-            if [[ $line = filenames ]]; then
+            if [[ $completion = filenames ]]; then
                 compopt -o filenames
             fi
         else
-            COMPREPLY+=("$line")
+            COMPREPLY+=("$completion")
         fi
     done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
     __ltrim_colon_completions "$cur"
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 3411e2d7a..9151a0344 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
                 globalConfig.getSettings(settings);
                 for (auto & s : settings)
                     if (hasPrefix(s.first, prefix))
-                        completions->insert(s.first);
+                        completions->add(s.first, s.second.description);
             }
         }
     });
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 2760b830b..66d8df085 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -17,8 +17,19 @@ void Args::addFlag(Flag && flag_)
     if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
+void Completions::add(std::string completion, std::string description)
+{
+    insert(Completion{
+            .completion = completion,
+            .description = description
+            });
+}
+
+bool Completion::operator<(const Completion & other) const
+{ return completion < other.completion || (completion == other.completion && description < other.description); }
+
 bool pathCompletions = false;
-std::shared_ptr<std::set<std::string>> completions;
+std::shared_ptr<Completions> completions;
 
 std::string completionMarker = "___COMPLETE___";
 
@@ -148,7 +159,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
             for (auto & [name, flag] : longFlags) {
                 if (!hiddenCategories.count(flag->category)
                     && hasPrefix(name, std::string(*prefix, 2)))
-                    completions->insert("--" + name);
+                    completions->add("--" + name, flag->description);
             }
         }
         auto i = longFlags.find(string(*pos, 2));
@@ -165,9 +176,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 
     if (auto prefix = needsCompletion(*pos)) {
         if (prefix == "-") {
-            completions->insert("--");
-            for (auto & [flag, _] : shortFlags)
-                completions->insert(std::string("-") + flag);
+            completions->add("--");
+            for (auto & [flagName, flag] : shortFlags)
+                completions->add(std::string("-") + flagName, flag->description);
         }
     }
 
@@ -244,11 +255,11 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashTypeCompleter(size_t index, std::string_view prefix) 
+static void hashTypeCompleter(size_t index, std::string_view prefix)
 {
     for (auto & type : hashTypes)
         if (hasPrefix(type, prefix))
-            completions->insert(type);
+            completions->add(type);
 }
 
 Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
@@ -292,7 +303,7 @@ static void _completePath(std::string_view prefix, bool onlyDirs)
                 auto st = lstat(globbuf.gl_pathv[i]);
                 if (!S_ISDIR(st.st_mode)) continue;
             }
-            completions->insert(globbuf.gl_pathv[i]);
+            completions->add(globbuf.gl_pathv[i]);
         }
         globfree(&globbuf);
     }
@@ -385,7 +396,7 @@ MultiCommand::MultiCommand(const Commands & commands)
             if (auto prefix = needsCompletion(s)) {
                 for (auto & [name, command] : commands)
                     if (hasPrefix(name, *prefix))
-                        completions->insert(name);
+                        completions->add(name);
             }
             auto i = commands.find(s);
             if (i == commands.end())
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index f41242e17..26f1bc11b 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -283,7 +283,17 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
 
 void printTable(std::ostream & out, const Table2 & table);
 
-extern std::shared_ptr<std::set<std::string>> completions;
+struct Completion {
+    std::string completion;
+    std::string description;
+
+    bool operator<(const Completion & other) const;
+};
+class Completions : public std::set<Completion> {
+public:
+    void add(std::string completion, std::string description = "");
+};
+extern std::shared_ptr<Completions> completions;
 extern bool pathCompletions;
 
 std::optional<std::string> needsCompletion(std::string_view s);
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 9bf6b7caa..7473c9758 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -26,7 +26,7 @@ void completeFlakeInputPath(
     auto flake = flake::getFlake(*evalState, flakeRef, true);
     for (auto & input : flake.inputs)
         if (hasPrefix(input.first, prefix))
-            completions->insert(input.first);
+            completions->add(input.first);
 }
 
 MixFlakeOptions::MixFlakeOptions()
@@ -211,7 +211,7 @@ void completeFlakeRefWithFragment(
                         auto attrPath2 = attr->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
-                        completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2));
+                        completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
                     }
                 }
             }
@@ -222,7 +222,7 @@ void completeFlakeRefWithFragment(
                 for (auto & attrPath : defaultFlakeAttrPaths) {
                     auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
                     if (!attr) continue;
-                    completions->insert(flakeRefS + "#");
+                    completions->add(flakeRefS + "#");
                 }
             }
         }
@@ -243,7 +243,7 @@ ref<EvalState> EvalCommand::getEvalState()
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")
-        completions->insert(".");
+        completions->add(".");
 
     completeDir(0, prefix);
 
@@ -254,10 +254,10 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
                 std::string from2(from, 6);
                 if (hasPrefix(from2, prefix))
-                    completions->insert(from2);
+                    completions->add(from2);
             } else {
                 if (hasPrefix(from, prefix))
-                    completions->insert(from);
+                    completions->add(from);
             }
         }
     }
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 1e9e07bc0..5056ceb78 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -208,7 +208,7 @@ void mainWrapped(int argc, char * * argv)
         if (completions) {
             std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n");
             for (auto & s : *completions)
-                std::cout << s << "\n";
+                std::cout << s.completion << "\t" << s.description << "\n";
         }
     });
 

From eea310b2418e30280ae5fd9dac54ffe7f3c5dafa Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 9 Oct 2020 09:41:55 +0200
Subject: [PATCH 345/882] Add a zsh completion script

Based on @clhodapp's suggestion in https://github.com/spwhitt/nix-zsh-completions/issues/32#issuecomment-705315356 and adapted to use the description of the completions
---
 misc/zsh/completion.zsh | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 misc/zsh/completion.zsh

diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh
new file mode 100644
index 000000000..d4df6447e
--- /dev/null
+++ b/misc/zsh/completion.zsh
@@ -0,0 +1,21 @@
+function _nix() {
+  local ifs_bk="$IFS"
+  local input=("${(Q)words[@]}")
+  IFS=$'\n'
+  local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]"))
+  IFS="$ifs_bk"
+  local tpe="${${res[1]}%%>	*}"
+  local -a suggestions
+  declare -a suggestions
+  for suggestion in ${res:1}; do
+    # FIXME: This doesn't work properly if the suggestion word contains a `:`
+    # itself
+    suggestions+="${suggestion/	/:}"
+  done
+  if [[ "$tpe" == filenames ]]; then
+    compadd -f
+  fi
+  _describe 'nix' suggestions
+}
+
+compdef _nix nix

From 636ec171391660e986623cb75c400a99d652c991 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Oct 2020 15:41:24 +0200
Subject: [PATCH 346/882] Remove stray DerivationOutputsAndPaths type

---
 src/libstore/derivations.hh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index d48266774..6d292b2e5 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -61,8 +61,6 @@ typedef std::map<string, DerivationOutput> DerivationOutputs;
    also contains, for each output, the (optional) store path in which it would
    be written. To calculate values of these types, see the corresponding
    functions in BasicDerivation */
-typedef std::map<string, std::pair<DerivationOutput, StorePath>>
-  DerivationOutputsAndPaths;
 typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
   DerivationOutputsAndOptPaths;
 

From 87157b2bd3224b8329ffcb73e92c64bdc36cff16 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Oct 2020 16:02:53 +0200
Subject: [PATCH 347/882] writeFile(): Add error context to writeFull() failure

Issue #4092.
---
 src/libutil/util.cc | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index f07e99885..1a8873136 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -326,7 +326,12 @@ void writeFile(const Path & path, const string & s, mode_t mode)
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
         throw SysError("opening file '%1%'", path);
-    writeFull(fd.get(), s);
+    try {
+        writeFull(fd.get(), s);
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", path);
+        throw;
+    }
 }
 
 
@@ -338,11 +343,16 @@ void writeFile(const Path & path, Source & source, mode_t mode)
 
     std::vector<unsigned char> buf(64 * 1024);
 
-    while (true) {
-        try {
-            auto n = source.read(buf.data(), buf.size());
-            writeFull(fd.get(), (unsigned char *) buf.data(), n);
-        } catch (EndOfFile &) { break; }
+    try {
+        while (true) {
+            try {
+                auto n = source.read(buf.data(), buf.size());
+                writeFull(fd.get(), (unsigned char *) buf.data(), n);
+            } catch (EndOfFile &) { break; }
+        }
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", path);
+        throw;
     }
 }
 

From e845d19ae368cb9ee6371c4b2fdbdc86a110d893 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Oct 2020 17:54:59 +0200
Subject: [PATCH 348/882] Remove Lazy

This fixes a crash during startup when compiling Nix as a single
compilation unit.
---
 src/libutil/lazy.hh | 48 ---------------------------------------------
 src/libutil/util.cc | 34 +++++++++++++++++---------------
 2 files changed, 18 insertions(+), 64 deletions(-)
 delete mode 100644 src/libutil/lazy.hh

diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh
deleted file mode 100644
index d073e486c..000000000
--- a/src/libutil/lazy.hh
+++ /dev/null
@@ -1,48 +0,0 @@
-#include <exception>
-#include <functional>
-#include <mutex>
-
-namespace nix {
-
-/* A helper class for lazily-initialized variables.
-
-     Lazy<T> var([]() { return value; });
-
-   declares a variable of type T that is initialized to 'value' (in a
-   thread-safe way) on first use, that is, when var() is first
-   called. If the initialiser code throws an exception, then all
-   subsequent calls to var() will rethrow that exception. */
-template<typename T>
-class Lazy
-{
-
-    typedef std::function<T()> Init;
-
-    Init init;
-
-    std::once_flag done;
-
-    T value;
-
-    std::exception_ptr ex;
-
-public:
-
-    Lazy(Init init) : init(init)
-    { }
-
-    const T & operator () ()
-    {
-        std::call_once(done, [&]() {
-            try {
-                value = init();
-            } catch (...) {
-                ex = std::current_exception();
-            }
-        });
-        if (ex) std::rethrow_exception(ex);
-        return value;
-    }
-};
-
-}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1a8873136..9804e9a51 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,4 +1,3 @@
-#include "lazy.hh"
 #include "util.hh"
 #include "affinity.hh"
 #include "sync.hh"
@@ -522,21 +521,24 @@ std::string getUserName()
 }
 
 
-static Lazy<Path> getHome2([]() {
-    auto homeDir = getEnv("HOME");
-    if (!homeDir) {
-        std::vector<char> buf(16384);
-        struct passwd pwbuf;
-        struct passwd * pw;
-        if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
-            || !pw || !pw->pw_dir || !pw->pw_dir[0])
-            throw Error("cannot determine user's home directory");
-        homeDir = pw->pw_dir;
-    }
-    return *homeDir;
-});
-
-Path getHome() { return getHome2(); }
+Path getHome()
+{
+    static Path homeDir = []()
+    {
+        auto homeDir = getEnv("HOME");
+        if (!homeDir) {
+            std::vector<char> buf(16384);
+            struct passwd pwbuf;
+            struct passwd * pw;
+            if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
+                || !pw || !pw->pw_dir || !pw->pw_dir[0])
+                throw Error("cannot determine user's home directory");
+            homeDir = pw->pw_dir;
+        }
+        return *homeDir;
+    }();
+    return homeDir;
+}
 
 
 Path getCacheDir()

From 59bd6e87a44fab86c511f78ff50f3e3c4cca4b59 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Oct 2020 21:55:59 +0200
Subject: [PATCH 349/882] Completions::add(): Guard against newlines

---
 src/libutil/args.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 66d8df085..8bd9c8aeb 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -19,10 +19,11 @@ void Args::addFlag(Flag && flag_)
 
 void Completions::add(std::string completion, std::string description)
 {
-    insert(Completion{
-            .completion = completion,
-            .description = description
-            });
+    assert(description.find('\n') == std::string::npos);
+    insert(Completion {
+        .completion = completion,
+        .description = description
+    });
 }
 
 bool Completion::operator<(const Completion & other) const

From 44349064f75616fd47194425e27f5cb1d6ef5ecb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Oct 2020 22:02:00 +0200
Subject: [PATCH 350/882] nix develop: Source ~/.bashrc

Fixes #4104.
---
 src/nix/develop.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index a46ea39b6..f763e8ba0 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -377,6 +377,10 @@ struct CmdDevelop : Common, MixEnvironment
             script += fmt("exec %s\n", concatStringsSep(" ", args));
         }
 
+        else {
+            script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n";
+        }
+
         writeFull(rcFileFd.get(), script);
 
         stopProgressBar();

From 725488b8922c731042a557608b8d833b05e77243 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Oct 2020 22:03:18 +0200
Subject: [PATCH 351/882] nix develop: Unset $HOSTNAME

This is set to "localhost" by stdenv which is probably not what you
want.
---
 src/nix/develop.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index f763e8ba0..9372f43de 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -164,6 +164,7 @@ struct Common : InstallableCommand, MixProfile
         "BASHOPTS",
         "EUID",
         "HOME", // FIXME: don't ignore in pure mode?
+        "HOSTNAME",
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",

From 6cc1541782084111a8fa0a1e34d685a2e8c58459 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 9 Oct 2020 20:18:08 +0000
Subject: [PATCH 352/882] Split out `local-fs-store.hh`

This matches the already-existing `local-fs-store.cc`.
---
 src/libfetchers/registry.cc            |  1 +
 src/libstore/gc.cc                     |  1 +
 src/libstore/local-fs-store.cc         |  1 +
 src/libstore/local-fs-store.hh         | 48 ++++++++++++++++++++++++++
 src/libstore/local-store.hh            |  1 +
 src/libstore/profiles.cc               |  1 +
 src/libstore/remote-store.hh           |  1 +
 src/libstore/store-api.hh              | 41 ----------------------
 src/nix-build/nix-build.cc             |  1 +
 src/nix-env/nix-env.cc                 |  1 +
 src/nix-env/user-env.cc                |  1 +
 src/nix-instantiate/nix-instantiate.cc |  1 +
 src/nix/build.cc                       |  1 +
 src/nix/bundle.cc                      |  1 +
 src/nix/command.cc                     |  1 +
 src/nix/doctor.cc                      |  1 +
 16 files changed, 62 insertions(+), 41 deletions(-)
 create mode 100644 src/libstore/local-fs-store.hh

diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 4367ee810..2426882ca 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -3,6 +3,7 @@
 #include "util.hh"
 #include "globals.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 518a357ef..7a04b2f89 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -1,6 +1,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "local-store.hh"
+#include "local-fs-store.hh"
 #include "finally.hh"
 
 #include <functional>
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 2f1d9663a..e7c3dae92 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -1,6 +1,7 @@
 #include "archive.hh"
 #include "fs-accessor.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "globals.hh"
 #include "compression.hh"
 #include "derivations.hh"
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
new file mode 100644
index 000000000..8eccd8236
--- /dev/null
+++ b/src/libstore/local-fs-store.hh
@@ -0,0 +1,48 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+struct LocalFSStoreConfig : virtual StoreConfig
+{
+    using StoreConfig::StoreConfig;
+    // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
+    // it to omit the call to the Setting constructor. Clang works fine
+    // either way.
+    const PathSetting rootDir{(StoreConfig*) this, true, "",
+        "root", "directory prefixed to all other paths"};
+    const PathSetting stateDir{(StoreConfig*) this, false,
+        rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
+        "state", "directory where Nix will store state"};
+    const PathSetting logDir{(StoreConfig*) this, false,
+        rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
+        "log", "directory where Nix will store state"};
+};
+
+class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
+{
+public:
+
+    const static string drvsLogDir;
+
+    LocalFSStore(const Params & params);
+
+    void narFromPath(const StorePath & path, Sink & sink) override;
+    ref<FSAccessor> getFSAccessor() override;
+
+    /* Register a permanent GC root. */
+    Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
+
+    virtual Path getRealStoreDir() { return storeDir; }
+
+    Path toRealPath(const Path & storePath) override
+    {
+        assert(isInStore(storePath));
+        return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
+    }
+
+    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
+};
+
+}
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index e7c9d1605..f1e2ab7f9 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -4,6 +4,7 @@
 
 #include "pathlocks.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "sync.hh"
 #include "util.hh"
 
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index c3809bad7..ed10dd519 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -1,5 +1,6 @@
 #include "profiles.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "util.hh"
 
 #include <sys/types.h>
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index ec04be985..554fb6bed 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -4,6 +4,7 @@
 #include <string>
 
 #include "store-api.hh"
+#include "local-fs-store.hh"
 
 
 namespace nix {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 450c0f554..f77bc21d1 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -715,47 +715,6 @@ protected:
 
 };
 
-struct LocalFSStoreConfig : virtual StoreConfig
-{
-    using StoreConfig::StoreConfig;
-    // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
-    // it to omit the call to the Setting constructor. Clang works fine
-    // either way.
-    const PathSetting rootDir{(StoreConfig*) this, true, "",
-        "root", "directory prefixed to all other paths"};
-    const PathSetting stateDir{(StoreConfig*) this, false,
-        rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
-        "state", "directory where Nix will store state"};
-    const PathSetting logDir{(StoreConfig*) this, false,
-        rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
-        "log", "directory where Nix will store state"};
-};
-
-class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
-{
-public:
-
-    const static string drvsLogDir;
-
-    LocalFSStore(const Params & params);
-
-    void narFromPath(const StorePath & path, Sink & sink) override;
-    ref<FSAccessor> getFSAccessor() override;
-
-    /* Register a permanent GC root. */
-    Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
-
-    virtual Path getRealStoreDir() { return storeDir; }
-
-    Path toRealPath(const Path & storePath) override
-    {
-        assert(isInStore(storePath));
-        return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
-    }
-
-    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
-};
-
 
 /* Copy a path from one store to another. */
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index a79b1086b..f60e0706c 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -6,6 +6,7 @@
 #include <vector>
 
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "globals.hh"
 #include "derivations.hh"
 #include "affinity.hh"
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index e6667e7f5..a4b5c9e2c 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -8,6 +8,7 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "user-env.hh"
 #include "util.hh"
 #include "json.hh"
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 8c6c8af05..87387e794 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "globals.hh"
 #include "shared.hh"
 #include "eval.hh"
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 18a0049a6..3956fef6d 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -8,6 +8,7 @@
 #include "value-to-json.hh"
 #include "util.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "common-eval-args.hh"
 #include "../nix/legacy.hh"
 
diff --git a/src/nix/build.cc b/src/nix/build.cc
index d85a482db..65708e98b 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -3,6 +3,7 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 
 using namespace nix;
 
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index fc41da9e4..2d0a0b6ea 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -2,6 +2,7 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "fs-accessor.hh"
 
 using namespace nix;
diff --git a/src/nix/command.cc b/src/nix/command.cc
index ba7de9fdd..9a38c77f1 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -1,5 +1,6 @@
 #include "command.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "derivations.hh"
 #include "nixexpr.hh"
 #include "profiles.hh"
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 4588ac05e..4f3003448 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -5,6 +5,7 @@
 #include "serve-protocol.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
 

From aef44cbaa9aaa6fa40eb34c411cedd858e16101e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:38:46 +0000
Subject: [PATCH 353/882] Split out `commonChildInit`

---
 src/libstore/build.cc | 29 -----------------------------
 src/libutil/util.cc   | 29 +++++++++++++++++++++++++++++
 src/libutil/util.hh   |  2 ++
 3 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 05a9ef088..5757e5c3f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -456,35 +456,6 @@ void Goal::trace(const FormatOrString & fs)
 //////////////////////////////////////////////////////////////////////
 
 
-/* Common initialisation performed in child processes. */
-static void commonChildInit(Pipe & logPipe)
-{
-    restoreSignals();
-
-    /* Put the child in a separate session (and thus a separate
-       process group) so that it has no controlling terminal (meaning
-       that e.g. ssh cannot open /dev/tty) and it doesn't receive
-       terminal signals. */
-    if (setsid() == -1)
-        throw SysError("creating a new session");
-
-    /* Dup the write side of the logger pipe into stderr. */
-    if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
-        throw SysError("cannot pipe standard error into log file");
-
-    /* Dup stderr to stdout. */
-    if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
-        throw SysError("cannot dup stderr into stdout");
-
-    /* Reroute stdin to /dev/null. */
-    int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
-    if (fdDevNull == -1)
-        throw SysError("cannot open '%1%'", pathNullDevice);
-    if (dup2(fdDevNull, STDIN_FILENO) == -1)
-        throw SysError("cannot dup null device into stdin");
-    close(fdDevNull);
-}
-
 void handleDiffHook(
     uid_t uid, uid_t gid,
     const Path & tryA, const Path & tryB,
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 9804e9a51..53342b5cb 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1660,4 +1660,33 @@ string showBytes(uint64_t bytes)
 }
 
 
+void commonChildInit(Pipe & logPipe)
+{
+    const static string pathNullDevice = "/dev/null";
+    restoreSignals();
+
+    /* Put the child in a separate session (and thus a separate
+       process group) so that it has no controlling terminal (meaning
+       that e.g. ssh cannot open /dev/tty) and it doesn't receive
+       terminal signals. */
+    if (setsid() == -1)
+        throw SysError("creating a new session");
+
+    /* Dup the write side of the logger pipe into stderr. */
+    if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
+        throw SysError("cannot pipe standard error into log file");
+
+    /* Dup stderr to stdout. */
+    if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
+        throw SysError("cannot dup stderr into stdout");
+
+    /* Reroute stdin to /dev/null. */
+    int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
+    if (fdDevNull == -1)
+        throw SysError("cannot open '%1%'", pathNullDevice);
+    if (dup2(fdDevNull, STDIN_FILENO) == -1)
+        throw SysError("cannot dup null device into stdin");
+    close(fdDevNull);
+}
+
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 129d59a97..cafe93702 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -536,6 +536,8 @@ typedef std::function<bool(const Path & path)> PathFilter;
 
 extern PathFilter defaultPathFilter;
 
+/* Common initialisation performed in child processes. */
+void commonChildInit(Pipe & logPipe);
 
 /* Create a Unix domain socket in listen mode. */
 AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);

From 428536fd75a8318a5b9a5274f071d4beb39d58dc Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:39:08 +0000
Subject: [PATCH 354/882] Prepare for build/* files

---
 src/libstore/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index d266c8efe..212a725af 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -4,7 +4,7 @@ libstore_NAME = libnixstore
 
 libstore_DIR := $(d)
 
-libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
+libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
 libstore_LIBS = libutil
 

From fc72cb07601dde8266ff2afd57f30381a66bb4c3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:16:29 +0000
Subject: [PATCH 355/882] Rename to hand-hold git (build.hh)

---
 src/libstore/{build.cc => build.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build.hh} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build.hh
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build.hh

From a4f0fecb032ae074f4d2395e2740b393ee740849 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:17:24 +0000
Subject: [PATCH 356/882] Trim build.hh

---
 src/libstore/build.hh | 4914 +----------------------------------------
 1 file changed, 22 insertions(+), 4892 deletions(-)

diff --git a/src/libstore/build.hh b/src/libstore/build.hh
index 5757e5c3f..8027c61b1 100644
--- a/src/libstore/build.hh
+++ b/src/libstore/build.hh
@@ -1,24 +1,11 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
+#pragma once
 
+#include "machines.hh"
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "local-store.hh"
+
+#include <memory>
 #include <algorithm>
 #include <iostream>
 #include <map>
@@ -28,60 +15,12 @@
 #include <chrono>
 #include <regex>
 #include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
 
 namespace nix {
 
 using std::map;
 
 
-static string pathNullDevice = "/dev/null";
-
-
 /* Forward definition. */
 class Worker;
 struct HookInstance;
@@ -184,14 +123,6 @@ struct Goal : public std::enable_shared_from_this<Goal>
     void amDone(ExitCode result, std::optional<Error> ex = {});
 };
 
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
 
@@ -372,332 +303,6 @@ public:
     }
 };
 
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
@@ -1072,3648 +677,6 @@ private:
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 class SubstitutionGoal : public Goal
 {
     friend class Worker;
@@ -4792,863 +755,30 @@ public:
     StorePath getStorePath() { return storePath; }
 };
 
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
+struct HookInstance
 {
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
+    /* Pipes for talking to the build hook. */
+    Pipe toHook;
 
+    /* Pipe for the hook's standard output/error. */
+    Pipe fromHook;
 
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
+    /* Pipe for the builder's standard output/error. */
+    Pipe builderOut;
 
+    /* The process ID of the hook. */
+    Pid pid;
 
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
+    FdSink sink;
 
+    std::map<ActivityId, Activity> activities;
 
-void SubstitutionGoal::init()
-{
-    trace("init");
+    HookInstance();
 
-    worker.store.addTempRoot(storePath);
+    ~HookInstance();
+};
 
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
 
+void addToWeakGoals(WeakGoals & goals, GoalPtr p);
 
 }

From 9629290eda0828f842a9a5d1965c237d921091f8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:18:50 +0000
Subject: [PATCH 357/882] Rename to hand-hold git (build/derivation-goal.cc)

---
 src/libstore/{build.cc => build/derivation-goal.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build/derivation-goal.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build/derivation-goal.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build/derivation-goal.cc

From 3bab1c5bb0a56f850a7bc1bacc9f974b108cf601 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:19:10 +0000
Subject: [PATCH 358/882] Trim build/derivation-goal.cc

---
 src/libstore/build/derivation-goal.cc | 1960 +------------------------
 1 file changed, 15 insertions(+), 1945 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 5757e5c3f..bb8921759 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1,53 +1,31 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
+#include "build.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
-#include "filetransfer.hh"
+#include "references.hh"
 #include "finally.hh"
-#include "compression.hh"
+#include "util.hh"
+#include "archive.hh"
 #include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
+#include "compression.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
 #include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include <fcntl.h>
 #include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
+#include <fcntl.h>
 #include <termios.h>
-#include <poll.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
 
-#include <pwd.h>
-#include <grp.h>
+#if HAVE_STATVFS
+#include <sys/statvfs.h>
+#endif
 
 /* Includes required for chroot support. */
 #if __linux__
@@ -67,395 +45,13 @@
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
+#include <pwd.h>
+#include <grp.h>
 
 #include <nlohmann/json.hpp>
 
-
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 void handleDiffHook(
     uid_t uid, uid_t gid,
     const Path & tryA, const Path & tryB,
@@ -488,594 +84,8 @@ void handleDiffHook(
     }
 }
 
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
 const Path DerivationGoal::homeDir = "/homeless-shelter";
 
-
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker)
@@ -4711,944 +3721,4 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
 }
 
 
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From 184bfc301e19ce9b11ace6ae0bd5a753da4e2931 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:19:55 +0000
Subject: [PATCH 359/882] Rename to hand-hold git (build/goal.cc)

---
 src/libstore/{build.cc => build/goal.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build/goal.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build/goal.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build/goal.cc

From 819fe848ac7a1fe88af5b648266a356c86224586 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:20:28 +0000
Subject: [PATCH 360/882] Trim build/goal.cc

---
 src/libstore/build/goal.cc | 5568 +-----------------------------------
 1 file changed, 1 insertion(+), 5567 deletions(-)

diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index 5757e5c3f..404a430f7 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -1,189 +1,7 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
+#include "build.hh"
 
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
 
 bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
     string s1 = a->key();
@@ -192,190 +10,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
 }
 
 
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 void addToWeakGoals(WeakGoals & goals, GoalPtr p)
 {
     // FIXME: necessary?
@@ -451,5204 +85,4 @@ void Goal::trace(const FormatOrString & fs)
     debug("%1%: %2%", name, fs.s);
 }
 
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From f0b8987299285c398a6db2ca32f3007a441c3a90 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:21:30 +0000
Subject: [PATCH 361/882] Rename to hand-hold git (build/hook-instance.cc)

---
 src/libstore/{build.cc => build/hook-instance.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build/hook-instance.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build/hook-instance.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build/hook-instance.cc

From 159054f7305c2161f78074570232199e160dad57 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:21:54 +0000
Subject: [PATCH 362/882] Trim build/hook-instance.cc

---
 src/libstore/build/hook-instance.cc | 5585 +--------------------------
 1 file changed, 1 insertion(+), 5584 deletions(-)

diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
index 5757e5c3f..a33d5e22b 100644
--- a/src/libstore/build/hook-instance.cc
+++ b/src/libstore/build/hook-instance.cc
@@ -1,633 +1,7 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
+#include "build.hh"
 
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
 HookInstance::HookInstance()
 {
     debug("starting build hook '%s'", settings.buildHook);
@@ -694,4961 +68,4 @@ HookInstance::~HookInstance()
     }
 }
 
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From dc5225cde5aa1cd240483caf08f4efdaab11ddae Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:25:04 +0000
Subject: [PATCH 363/882] Rename to hand-hold git (build/local-store-build.cc)

---
 src/libstore/{build.cc => build/local-store-build.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build/local-store-build.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build/local-store-build.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build/local-store-build.cc

From 4bdff7d1b0040dee7fdd51b5338f79290375c490 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:25:25 +0000
Subject: [PATCH 364/882] Trim build/local-store-build.cc

---
 src/libstore/build/local-store-build.cc | 5533 +----------------------
 1 file changed, 1 insertion(+), 5532 deletions(-)

diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc
index 5757e5c3f..407979b1e 100644
--- a/src/libstore/build/local-store-build.cc
+++ b/src/libstore/build/local-store-build.cc
@@ -1,5537 +1,7 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
+#include "build.hh"
 
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
 {
     StorePathSet willBuild, willSubstitute, unknown;
@@ -5650,5 +120,4 @@ void LocalStore::repairPath(const StorePath & path)
     }
 }
 
-
 }

From 3633b3572b66c4fdbbff1f578687bf5926da969b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:25:58 +0000
Subject: [PATCH 365/882] Rename to hand-hold git (build/substitution-goal.cc)

---
 src/libstore/{build.cc => build/substitution-goal.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build/substitution-goal.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build/substitution-goal.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build/substitution-goal.cc

From d24ffe0eb146cbc48b9006415c795ad38efd6db0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:26:10 +0000
Subject: [PATCH 366/882] Trim build/substitution-goal.cc

---
 src/libstore/build/substitution-goal.cc | 5363 +----------------------
 1 file changed, 2 insertions(+), 5361 deletions(-)

diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 5757e5c3f..dd2ce94a6 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -1,4798 +1,9 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
+#include "build.hh"
 #include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
+#include "finally.hh"
 
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
 SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
     : Goal(worker)
     , storePath(storePath)
@@ -5081,574 +292,4 @@ void SubstitutionGoal::handleEOF(int fd)
     if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }
 
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From 904e315dae7a937465b762a0c0e8ccb4ccd27dc0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:26:54 +0000
Subject: [PATCH 367/882] Rename to hand-hold git (build/worker.cc)

---
 src/libstore/{build.cc => build/worker.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => build/worker.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/build/worker.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/build/worker.cc

From eed53ed87aed55d8a9ad7d0edae6ef37f14b9988 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:28:49 +0000
Subject: [PATCH 368/882] Trim build/worker.cc

---
 src/libstore/build/worker.cc | 5205 +---------------------------------
 1 file changed, 1 insertion(+), 5204 deletions(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 5757e5c3f..2fc9f6982 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -1,5089 +1,9 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
+#include "build.hh"
 
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
 #include <poll.h>
 
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
-
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
 Worker::Worker(LocalStore & store)
     : act(*logger, actRealise)
     , actDerivations(*logger, actBuilds)
@@ -5528,127 +448,4 @@ void Worker::markContentsGood(const StorePath & path)
     pathContentsGoodCache.insert_or_assign(path, true);
 }
 
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From d0004bfcab0741e76c65f28e46b9c06a7979d49c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:33:18 +0000
Subject: [PATCH 369/882] Rename to hand-hold git (lock.hh)

---
 src/libstore/{build.cc => lock.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => lock.hh} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/lock.hh
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/lock.hh

From dbc588651c2e2e8952852e390c1d09f61a275f94 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:34:36 +0000
Subject: [PATCH 370/882] Trim lock.hh

---
 src/libstore/lock.hh | 5625 +-----------------------------------------
 1 file changed, 4 insertions(+), 5621 deletions(-)

diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh
index 5757e5c3f..8fbb67ddc 100644
--- a/src/libstore/lock.hh
+++ b/src/libstore/lock.hh
@@ -1,496 +1,11 @@
-#include "references.hh"
-#include "pathlocks.hh"
-#include "globals.hh"
-#include "local-store.hh"
+#pragma once
+
+#include "sync.hh"
+#include "types.hh"
 #include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
-#include <grp.h>
-
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
 
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
 class UserLock
 {
 private:
@@ -519,5136 +34,4 @@ public:
 
 };
 
-
-UserLock::UserLock()
-{
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    killUser(uid);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From e0be04129b058680b3b3c64098cf3a0aa9d3901e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:35:06 +0000
Subject: [PATCH 371/882] Rename to hand-hold git (lock.cc)

---
 src/libstore/{build.cc => lock.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.cc => lock.cc} (100%)

diff --git a/src/libstore/build.cc b/src/libstore/lock.cc
similarity index 100%
rename from src/libstore/build.cc
rename to src/libstore/lock.cc

From bcb67e1ed8ac3d8eea8c9b7c2f69c9ea2ab8c12b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:35:19 +0000
Subject: [PATCH 372/882] Trim lock.cc

---
 src/libstore/lock.cc | 5571 +-----------------------------------------
 1 file changed, 5 insertions(+), 5566 deletions(-)

diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index 5757e5c3f..f1356fdca 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -1,525 +1,15 @@
-#include "references.hh"
-#include "pathlocks.hh"
+#include "lock.hh"
 #include "globals.hh"
-#include "local-store.hh"
-#include "util.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "builtins.hh"
-#include "builtins/buildenv.hh"
-#include "filetransfer.hh"
-#include "finally.hh"
-#include "compression.hh"
-#include "json.hh"
-#include "nar-info.hh"
-#include "parsed-derivations.hh"
-#include "machines.hh"
-#include "daemon.hh"
-#include "worker-protocol.hh"
-#include "topo-sort.hh"
-#include "callback.hh"
+#include "pathlocks.hh"
 
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
-#include <climits>
-
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <unistd.h>
-#include <errno.h>
-#include <cstring>
-#include <termios.h>
-#include <poll.h>
-
-#include <pwd.h>
 #include <grp.h>
+#include <pwd.h>
 
-/* Includes required for chroot support. */
-#if __linux__
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <net/if.h>
-#include <netinet/ip.h>
-#include <sys/personality.h>
-#include <sys/mman.h>
-#include <sched.h>
-#include <sys/param.h>
-#include <sys/mount.h>
-#include <sys/syscall.h>
-#if HAVE_SECCOMP
-#include <seccomp.h>
-#endif
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
-#endif
-
-#if HAVE_STATVFS
-#include <sys/statvfs.h>
-#endif
-
-#include <nlohmann/json.hpp>
-
+#include <fcntl.h>
+#include <unistd.h>
 
 namespace nix {
 
-using std::map;
-
-
-static string pathNullDevice = "/dev/null";
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
-    string s2 = b->key();
-    return s1 < s2;
-}
-
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p)
-{
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
-}
-
-
-void Goal::addWaitee(GoalPtr waitee)
-{
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.find(waitee) != waitees.end());
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
-        }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
-    }
-}
-
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
-{
-    trace("done");
-    assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
-    exitCode = result;
-
-    if (ex) {
-        if (!waiters.empty())
-            logError(ex->info());
-        else
-            this->ex = std::move(*ex);
-    }
-
-    for (auto & i : waiters) {
-        GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
-    }
-    waiters.clear();
-    worker.removeGoal(shared_from_this());
-}
-
-
-void Goal::trace(const FormatOrString & fs)
-{
-    debug("%1%: %2%", name, fs.s);
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
-            logError(ei);
-        }
-    }
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    string getUser() { return user; }
-    uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
 UserLock::UserLock()
 {
     assert(settings.buildUsersGroup != "");
@@ -600,5055 +90,4 @@ void UserLock::kill()
     killUser(uid);
 }
 
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
-    debug("starting build hook '%s'", settings.buildHook);
-
-    /* Create a pipe to get the output of the child. */
-    fromHook.create();
-
-    /* Create the communication pipes. */
-    toHook.create();
-
-    /* Create a pipe to get the output of the builder. */
-    builderOut.create();
-
-    /* Fork the hook. */
-    pid = startProcess([&]() {
-
-        commonChildInit(fromHook);
-
-        if (chdir("/") == -1) throw SysError("changing into /");
-
-        /* Dup the communication pipes. */
-        if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-
-        /* Use fd 4 for the builder's stdout/stderr. */
-        if (dup2(builderOut.writeSide.get(), 4) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        /* Hack: pass the read side of that fd to allow build-remote
-           to read SSH error messages. */
-        if (dup2(builderOut.readSide.get(), 5) == -1)
-            throw SysError("dupping builder's stdout/stderr");
-
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
-
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
-    });
-
-    pid.setSeparatePG(true);
-    fromHook.writeSide = -1;
-    toHook.readSide = -1;
-
-    sink = FdSink(toHook.writeSide.get());
-    std::map<std::string, Config::SettingInfo> settings;
-    globalConfig.getSettings(settings);
-    for (auto & setting : settings)
-        sink << 1 << setting.first << setting.second.value;
-    sink << 0;
-}
-
-
-HookInstance::~HookInstance()
-{
-    try {
-        toHook.writeSide = -1;
-        if (pid != -1) pid.kill();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-
-
-DerivationGoal::~DerivationGoal()
-{
-    /* Careful: we should never ever throw an exception from a
-       destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
-}
-
-
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
-void DerivationGoal::killChild()
-{
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(worker.makeSubstitutionGoal(drvPath));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv)));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(worker.makeSubstitutionGoal(i));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(worker.makeSubstitutionGoal(i, Repair));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-
-void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
-}
-
-
-static void chmod_(const Path & path, mode_t mode)
-{
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("setting permissions on '%s'", path);
-}
-
-
-/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
-   it's a directory and we're not root (to be able to update the
-   directory's parent link ".."). */
-static void movePath(const Path & src, const Path & dst)
-{
-    auto st = lstat(src);
-
-    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
-
-    if (changePerm)
-        chmod_(src, st.st_mode | S_IWUSR);
-
-    if (rename(src.c_str(), dst.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
-
-    if (changePerm)
-        chmod_(dst, st.st_mode);
-}
-
-
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
-{
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
-
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
-}
-
-
-MakeError(NotDeterministic, BuildError);
-
-
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
-
-    sandboxMountNamespace = -1;
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            deleteTmpDir(true);
-            done(BuildResult::Built);
-            return;
-        }
-
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    machineName = readLine(hook->fromHook.readSide.get());
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
-        }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-
-
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
-        dirsInChroot.clear();
-
-        for (auto i : dirs) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        pid_t tmp;
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    while (true) {
-        string msg = readLine(builderOut.readSide.get());
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
-
-void DerivationGoal::registerOutputs()
-{
-    /* When using a build hook, the build hook can register the output
-       as valid (by doing `nix-store --import').  If so we don't have
-       to do anything here.
-
-       We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
-     */
-    if (hook) {
-        bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
-                allValid = false;
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
-            continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
-
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
-        }
-        worker.store.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-
-
-void DerivationGoal::deleteTmpDir(bool force)
-{
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
-}
-
-
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        initialOutputs.insert_or_assign(i.first, info);
-    }
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
-    , storePath(storePath)
-    , repair(repair)
-    , ca(ca)
-{
-    state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
-    trace("created");
-    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
-}
-
-
-SubstitutionGoal::~SubstitutionGoal()
-{
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void SubstitutionGoal::init()
-{
-    trace("init");
-
-    worker.store.addTempRoot(storePath);
-
-    /* If the path already exists we're done. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
-        return;
-    }
-
-    if (settings.readOnlyMode)
-        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
-
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-
-    tryNext();
-}
-
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
-
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
-        }
-
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
-        }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
-        }
-        throw;
-    }
-
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
-
-    worker.updateProgress();
-
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
-    {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &SubstitutionGoal::referencesValid;
-}
-
-
-void SubstitutionGoal::referencesValid()
-{
-    trace("all references realised");
-
-    if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
-    }
-
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
-
-    state = &SubstitutionGoal::tryToRun;
-    worker.wakeUp(shared_from_this());
-}
-
-
-void SubstitutionGoal::tryToRun()
-{
-    trace("trying to run");
-
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        return;
-    }
-
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
-    worker.updateProgress();
-
-    outPipe.create();
-
-    promise = std::promise<void>();
-
-    thr = std::thread([this]() {
-        try {
-            /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
-
-            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
-            PushActivity pact(act.id);
-
-            copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
-
-            promise.set_value();
-        } catch (...) {
-            promise.set_exception(std::current_exception());
-        }
-    });
-
-    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
-
-    state = &SubstitutionGoal::finished;
-}
-
-
-void SubstitutionGoal::finished()
-{
-    trace("substitute finished");
-
-    thr.join();
-    worker.childTerminated(this);
-
-    try {
-        promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-
-        /* Cause the parent build to fail unless --fallback is given,
-           or the substitute has disappeared. The latter case behaves
-           the same as the substitute never having existed in the
-           first place. */
-        try {
-            throw;
-        } catch (SubstituteGone &) {
-        } catch (...) {
-            substituterFailed = true;
-        }
-
-        /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
-    }
-
-    worker.markContentsGood(storePath);
-
-    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
-
-    maintainRunningSubstitutions.reset();
-
-    maintainExpectedSubstitutions.reset();
-    worker.doneSubstitutions++;
-
-    if (maintainExpectedDownload) {
-        auto fileSize = maintainExpectedDownload->delta;
-        maintainExpectedDownload.reset();
-        worker.doneDownloadSize += fileSize;
-    }
-
-    worker.doneNarSize += maintainExpectedNar->delta;
-    maintainExpectedNar.reset();
-
-    worker.updateProgress();
-
-    amDone(ecSuccess);
-}
-
-
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
-{
-}
-
-
-void SubstitutionGoal::handleEOF(int fd)
-{
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-Worker::Worker(LocalStore & store)
-    : act(*logger, actRealise)
-    , actDerivations(*logger, actBuilds)
-    , actSubstitutions(*logger, actCopyPaths)
-    , store(store)
-{
-    /* Debugging: prevent recursive workers. */
-    nrLocalBuilds = 0;
-    lastWokenUp = steady_time_point::min();
-    permanentFailure = false;
-    timedOut = false;
-    hashMismatch = false;
-    checkMismatch = false;
-}
-
-
-Worker::~Worker()
-{
-    /* Explicitly get rid of all strong pointers now.  After this all
-       goals that refer to this worker should be gone.  (Otherwise we
-       are in trouble, since goals may call childTerminated() etc. in
-       their destructors). */
-    topGoals.clear();
-
-    assert(expectedSubstitutions == 0);
-    assert(expectedDownloadSize == 0);
-    assert(expectedNarSize == 0);
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
-    const StorePath & drvPath,
-    const StringSet & wantedOutputs,
-    std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
-{
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
-        goal = mkDrvGoal();
-        abstract_goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
-{
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
-    });
-}
-
-
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
-{
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
-    if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
-        goal_weak = goal;
-        wakeUp(goal);
-    }
-    return goal;
-}
-
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
-{
-    /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
-}
-
-
-void Worker::removeGoal(GoalPtr goal)
-{
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
-    if (topGoals.find(goal) != topGoals.end()) {
-        topGoals.erase(goal);
-        /* If a top-level goal failed, then kill all other goals
-           (unless keepGoing was set). */
-        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
-            topGoals.clear();
-    }
-
-    /* Wake up goals waiting for any goal to finish. */
-    for (auto & i : waitingForAnyGoal) {
-        GoalPtr goal = i.lock();
-        if (goal) wakeUp(goal);
-    }
-
-    waitingForAnyGoal.clear();
-}
-
-
-void Worker::wakeUp(GoalPtr goal)
-{
-    goal->trace("woken up");
-    addToWeakGoals(awake, goal);
-}
-
-
-unsigned Worker::getNrLocalBuilds()
-{
-    return nrLocalBuilds;
-}
-
-
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
-    bool inBuildSlot, bool respectTimeouts)
-{
-    Child child;
-    child.goal = goal;
-    child.goal2 = goal.get();
-    child.fds = fds;
-    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
-    child.inBuildSlot = inBuildSlot;
-    child.respectTimeouts = respectTimeouts;
-    children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
-}
-
-
-void Worker::childTerminated(Goal * goal, bool wakeSleepers)
-{
-    auto i = std::find_if(children.begin(), children.end(),
-        [&](const Child & child) { return child.goal2 == goal; });
-    if (i == children.end()) return;
-
-    if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
-    }
-
-    children.erase(i);
-
-    if (wakeSleepers) {
-
-        /* Wake up goals waiting for a build slot. */
-        for (auto & j : wantingToBuild) {
-            GoalPtr goal = j.lock();
-            if (goal) wakeUp(goal);
-        }
-
-        wantingToBuild.clear();
-    }
-}
-
-
-void Worker::waitForBuildSlot(GoalPtr goal)
-{
-    debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
-        wakeUp(goal); /* we can do it right away */
-    else
-        addToWeakGoals(wantingToBuild, goal);
-}
-
-
-void Worker::waitForAnyGoal(GoalPtr goal)
-{
-    debug("wait for any goal");
-    addToWeakGoals(waitingForAnyGoal, goal);
-}
-
-
-void Worker::waitForAWhile(GoalPtr goal)
-{
-    debug("wait for a while");
-    addToWeakGoals(waitingForAWhile, goal);
-}
-
-
-void Worker::run(const Goals & _topGoals)
-{
-    for (auto & i : _topGoals) topGoals.insert(i);
-
-    debug("entered goal loop");
-
-    while (1) {
-
-        checkInterrupt();
-
-        store.autoGC(false);
-
-        /* Call every wake goal (in the ordering established by
-           CompareGoalPtrs). */
-        while (!awake.empty() && !topGoals.empty()) {
-            Goals awake2;
-            for (auto & i : awake) {
-                GoalPtr goal = i.lock();
-                if (goal) awake2.insert(goal);
-            }
-            awake.clear();
-            for (auto & goal : awake2) {
-                checkInterrupt();
-                goal->work();
-                if (topGoals.empty()) break; // stuff may have been cancelled
-            }
-        }
-
-        if (topGoals.empty()) break;
-
-        /* Wait for input. */
-        if (!children.empty() || !waitingForAWhile.empty())
-            waitForInput();
-        else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error("unable to start any build; either increase '--max-jobs' "
-                            "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-                else
-                   throw Error("unable to start any build; remote machines may not have "
-                            "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-
-            }
-            assert(!awake.empty());
-        }
-    }
-
-    /* If --keep-going is not set, it's possible that the main goal
-       exited while some of its subgoals were still active.  But if
-       --keep-going *is* set, then they must all be finished now. */
-    assert(!settings.keepGoing || awake.empty());
-    assert(!settings.keepGoing || wantingToBuild.empty());
-    assert(!settings.keepGoing || children.empty());
-}
-
-void Worker::waitForInput()
-{
-    printMsg(lvlVomit, "waiting for children");
-
-    /* Process output from the file descriptors attached to the
-       children, namely log output and output path creation commands.
-       We also use this to detect child termination: if we get EOF on
-       the logger pipe of a build, we assume that the builder has
-       terminated. */
-
-    bool useTimeout = false;
-    long timeout = 0;
-    auto before = steady_time_point::clock::now();
-
-    /* If we're monitoring for silence on stdout/stderr, or if there
-       is a build timeout, then wait for input until the first
-       deadline for any child. */
-    auto nearest = steady_time_point::max(); // nearest deadline
-    if (settings.minFree.get() != 0)
-        // Periodicallty wake up to see if we need to run the garbage collector.
-        nearest = before + std::chrono::seconds(10);
-    for (auto & i : children) {
-        if (!i.respectTimeouts) continue;
-        if (0 != settings.maxSilentTime)
-            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (0 != settings.buildTimeout)
-            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
-    }
-    if (nearest != steady_time_point::max()) {
-        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
-        useTimeout = true;
-    }
-
-    /* If we are polling goals that are waiting for a lock, then wake
-       up after a few seconds at most. */
-    if (!waitingForAWhile.empty()) {
-        useTimeout = true;
-        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout = std::max(1L,
-            (long) std::chrono::duration_cast<std::chrono::seconds>(
-                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
-    } else lastWokenUp = steady_time_point::min();
-
-    if (useTimeout)
-        vomit("sleeping %d seconds", timeout);
-
-    /* Use select() to wait for the input side of any logger pipe to
-       become `available'.  Note that `available' (i.e., non-blocking)
-       includes EOF. */
-    std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
-    for (auto & i : children) {
-        for (auto & j : i.fds) {
-            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
-            fdToPollStatus[j] = pollStatus.size() - 1;
-        }
-    }
-
-    if (poll(pollStatus.data(), pollStatus.size(),
-            useTimeout ? timeout * 1000 : -1) == -1) {
-        if (errno == EINTR) return;
-        throw SysError("waiting for input");
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    /* Process all available file descriptors. FIXME: this is
-       O(children * fds). */
-    decltype(children)::iterator i;
-    for (auto j = children.begin(); j != children.end(); j = i) {
-        i = std::next(j);
-
-        checkInterrupt();
-
-        GoalPtr goal = j->goal.lock();
-        assert(goal);
-
-        set<int> fds2(j->fds);
-        std::vector<unsigned char> buffer(4096);
-        for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
-                ssize_t rd = ::read(k, buffer.data(), buffer.size());
-                // FIXME: is there a cleaner way to handle pt close
-                // than EIO? Is this even standard?
-                if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug("%1%: got EOF", goal->getName());
-                    goal->handleEOF(k);
-                    j->fds.erase(k);
-                } else if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError("%s: read failed", goal->getName());
-                } else {
-                    printMsg(lvlVomit, "%1%: read %2% bytes",
-                        goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
-                    j->lastOutput = after;
-                    goal->handleChildOutput(k, data);
-                }
-            }
-        }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
-    }
-
-    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
-        lastWokenUp = after;
-        for (auto & i : waitingForAWhile) {
-            GoalPtr goal = i.lock();
-            if (goal) wakeUp(goal);
-        }
-        waitingForAWhile.clear();
-    }
-}
-
-
-unsigned int Worker::exitStatus()
-{
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
-    unsigned int mask = 0;
-    bool buildFailure = permanentFailure || timedOut || hashMismatch;
-    if (buildFailure)
-        mask |= 0x04;  // 100
-    if (timedOut)
-        mask |= 0x01;  // 101
-    if (hashMismatch)
-        mask |= 0x02;  // 102
-    if (checkMismatch) {
-        mask |= 0x08;  // 104
-    }
-
-    if (mask)
-        mask |= 0x60;
-    return mask ? mask : 1;
-}
-
-
-bool Worker::pathContentsGood(const StorePath & path)
-{
-    auto i = pathContentsGoodCache.find(path);
-    if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo("checking path '%s'...", store.printStorePath(path));
-    auto info = store.queryPathInfo(path);
-    bool res;
-    if (!pathExists(store.printStorePath(path)))
-        res = false;
-    else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
-    }
-    pathContentsGoodCache.insert_or_assign(path, res);
-    if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
-    return res;
-}
-
-
-void Worker::markContentsGood(const StorePath & path)
-{
-    pathContentsGoodCache.insert_or_assign(path, true);
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    Worker worker(*this);
-
-    primeCache(*this, drvPaths);
-
-    Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
-    }
-
-    worker.run(goals);
-
-    StorePathSet failed;
-    std::optional<Error> ex;
-    for (auto & i : goals) {
-        if (i->ex) {
-            if (ex)
-                logError(i->ex->info());
-            else
-                ex = i->ex;
-        }
-        if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
-        }
-    }
-
-    if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
-        throw *ex;
-    } else if (!failed.empty()) {
-        if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
-    }
-}
-
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
-
-    BuildResult result;
-
-    try {
-        worker.run(Goals{goal});
-        result = goal->getResult();
-    } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
-        result.errorMsg = e.msg();
-    }
-
-    return result;
-}
-
-
-void LocalStore::ensurePath(const StorePath & path)
-{
-    /* If the path is already valid, we're done. */
-    if (isValidPath(path)) return;
-
-    primeCache(*this, {{path}});
-
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
-        } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
-    }
-}
-
-
-void LocalStore::repairPath(const StorePath & path)
-{
-    Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
-    Goals goals = {goal};
-
-    worker.run(goals);
-
-    if (goal->exitCode != Goal::ecSuccess) {
-        /* Since substituting the path didn't work, if we have a valid
-           deriver, then rebuild the deriver. */
-        auto info = queryPathInfo(path);
-        if (info->deriver && isValidPath(*info->deriver)) {
-            goals.clear();
-            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
-            worker.run(goals);
-        } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
-    }
-}
-
-
 }

From 5c74a6147b4b81dc5b173f190f02f6681ec4b0fe Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 17:07:14 +0000
Subject: [PATCH 373/882] Properly type the derivation and substitution goal
 maps

As a bonus, Worker::removeGoal is less inefficient.
---
 src/libstore/build.hh        | 11 +++++------
 src/libstore/build/worker.cc | 33 +++++++++++++++++----------------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/libstore/build.hh b/src/libstore/build.hh
index 8027c61b1..894feab5b 100644
--- a/src/libstore/build.hh
+++ b/src/libstore/build.hh
@@ -28,7 +28,6 @@ struct HookInstance;
 
 /* A pointer to a goal. */
 struct Goal;
-class DerivationGoal;
 typedef std::shared_ptr<Goal> GoalPtr;
 typedef std::weak_ptr<Goal> WeakGoalPtr;
 
@@ -140,6 +139,8 @@ struct Child
     steady_time_point timeStarted;
 };
 
+class DerivationGoal;
+class SubstitutionGoal;
 
 /* The worker class. */
 class Worker
@@ -167,8 +168,8 @@ private:
 
     /* Maps used to prevent multiple instantiations of a goal for the
        same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
+    std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
+    std::map<StorePath, std::weak_ptr<SubstitutionGoal>> substitutionGoals;
 
     /* Goals waiting for busy paths to be unlocked. */
     WeakGoals waitingForAnyGoal;
@@ -242,7 +243,7 @@ public:
         const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
 
     /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+    std::shared_ptr<SubstitutionGoal> makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
 
     /* Remove a dead goal. */
     void removeGoal(GoalPtr goal);
@@ -305,8 +306,6 @@ public:
 
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
-class SubstitutionGoal;
-
 /* Unless we are repairing, we don't both to test validity and just assume it,
    so the choices are `Absent` or `Valid`. */
 enum struct PathStatus {
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 2fc9f6982..47403580e 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -39,16 +39,13 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
     const StringSet & wantedOutputs,
     std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
 {
-    WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
-    GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
-    std::shared_ptr<DerivationGoal> goal;
-    if (!abstract_goal) {
+    std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
+    std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
+    if (!goal) {
         goal = mkDrvGoal();
-        abstract_goal_weak = goal;
+        goal_weak = goal;
         wakeUp(goal);
     } else {
-        goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
-        assert(goal);
         goal->addWantedOutputs(wantedOutputs);
     }
     return goal;
@@ -73,10 +70,10 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
 }
 
 
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::shared_ptr<SubstitutionGoal> Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
 {
-    WeakGoalPtr & goal_weak = substitutionGoals[path];
-    GoalPtr goal = goal_weak.lock(); // FIXME
+    std::weak_ptr<SubstitutionGoal> & goal_weak = substitutionGoals[path];
+    auto goal = goal_weak.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
         goal_weak = goal;
@@ -85,14 +82,14 @@ GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair,
     return goal;
 }
 
-
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
+template<typename G>
+static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
-    for (WeakGoalMap::iterator i = goalMap.begin();
+    for (typename std::map<StorePath, std::weak_ptr<G>>::iterator i = goalMap.begin();
          i != goalMap.end(); )
         if (i->second.lock() == goal) {
-            WeakGoalMap::iterator j = i; ++j;
+            typename std::map<StorePath, std::weak_ptr<G>>::iterator j = i; ++j;
             goalMap.erase(i);
             i = j;
         }
@@ -102,8 +99,12 @@ static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
 
 void Worker::removeGoal(GoalPtr goal)
 {
-    nix::removeGoal(goal, derivationGoals);
-    nix::removeGoal(goal, substitutionGoals);
+    if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
+        nix::removeGoal(drvGoal, derivationGoals);
+    else if (auto subGoal = std::dynamic_pointer_cast<SubstitutionGoal>(goal))
+        nix::removeGoal(subGoal, substitutionGoals);
+    else
+        assert(false);
     if (topGoals.find(goal) != topGoals.end()) {
         topGoals.erase(goal);
         /* If a top-level goal failed, then kill all other goals

From 38e3897162825e8b3a7005df140a04b9acf0a2fb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 17:18:26 +0000
Subject: [PATCH 374/882] Copy {uds-,}remote-store.{cc,hh}

This prepares for the splitting that happens in the next commit.
---
 src/libstore/uds-remote-store.cc | 1017 ++++++++++++++++++++++++++++++
 src/libstore/uds-remote-store.hh |  204 ++++++
 2 files changed, 1221 insertions(+)
 create mode 100644 src/libstore/uds-remote-store.cc
 create mode 100644 src/libstore/uds-remote-store.hh

diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
new file mode 100644
index 000000000..23b1942ce
--- /dev/null
+++ b/src/libstore/uds-remote-store.cc
@@ -0,0 +1,1017 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "remote-fs-accessor.hh"
+#include "remote-store.hh"
+#include "worker-protocol.hh"
+#include "archive.hh"
+#include "affinity.hh"
+#include "globals.hh"
+#include "derivations.hh"
+#include "pool.hh"
+#include "finally.hh"
+#include "logging.hh"
+#include "callback.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <cstring>
+
+namespace nix {
+
+namespace worker_proto {
+
+std::string read(const Store & store, Source & from, Phantom<std::string> _)
+{
+    return readString(from);
+}
+
+void write(const Store & store, Sink & out, const std::string & str)
+{
+    out << str;
+}
+
+
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
+{
+    return store.parseStorePath(readString(from));
+}
+
+void write(const Store & store, Sink & out, const StorePath & storePath)
+{
+    out << store.printStorePath(storePath);
+}
+
+
+ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
+{
+    return parseContentAddress(readString(from));
+}
+
+void write(const Store & store, Sink & out, const ContentAddress & ca)
+{
+    out << renderContentAddress(ca);
+}
+
+
+std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
+{
+    auto s = readString(from);
+    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
+}
+
+void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+{
+    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+}
+
+
+std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
+{
+    return parseContentAddressOpt(readString(from));
+}
+
+void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+{
+    out << (caOpt ? renderContentAddress(*caOpt) : "");
+}
+
+}
+
+
+/* TODO: Separate these store impls into different files, give them better names */
+RemoteStore::RemoteStore(const Params & params)
+    : Store(params)
+    , RemoteStoreConfig(params)
+    , connections(make_ref<Pool<Connection>>(
+            std::max(1, (int) maxConnections),
+            [this]() {
+                auto conn = openConnectionWrapper();
+                try {
+                    initConnection(*conn);
+                } catch (...) {
+                    failed = true;
+                    throw;
+                }
+                return conn;
+            },
+            [this](const ref<Connection> & r) {
+                return
+                    r->to.good()
+                    && r->from.good()
+                    && std::chrono::duration_cast<std::chrono::seconds>(
+                        std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
+            }
+            ))
+{
+}
+
+
+ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
+{
+    if (failed)
+        throw Error("opening a connection to remote store '%s' previously failed", getUri());
+    try {
+        return openConnection();
+    } catch (...) {
+        failed = true;
+        throw;
+    }
+}
+
+
+UDSRemoteStore::UDSRemoteStore(const Params & params)
+    : StoreConfig(params)
+    , Store(params)
+    , LocalFSStore(params)
+    , RemoteStore(params)
+{
+}
+
+
+UDSRemoteStore::UDSRemoteStore(
+        const std::string scheme,
+        std::string socket_path,
+        const Params & params)
+    : UDSRemoteStore(params)
+{
+    path.emplace(socket_path);
+}
+
+
+std::string UDSRemoteStore::getUri()
+{
+    if (path) {
+        return std::string("unix://") + *path;
+    } else {
+        return "daemon";
+    }
+}
+
+
+ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
+{
+    auto conn = make_ref<Connection>();
+
+    /* Connect to a daemon that does the privileged work for us. */
+    conn->fd = socket(PF_UNIX, SOCK_STREAM
+        #ifdef SOCK_CLOEXEC
+        | SOCK_CLOEXEC
+        #endif
+        , 0);
+    if (!conn->fd)
+        throw SysError("cannot create Unix domain socket");
+    closeOnExec(conn->fd.get());
+
+    string socketPath = path ? *path : settings.nixDaemonSocketFile;
+
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+    if (socketPath.size() + 1 >= sizeof(addr.sun_path))
+        throw Error("socket path '%1%' is too long", socketPath);
+    strcpy(addr.sun_path, socketPath.c_str());
+
+    if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
+        throw SysError("cannot connect to daemon at '%1%'", socketPath);
+
+    conn->from.fd = conn->fd.get();
+    conn->to.fd = conn->fd.get();
+
+    conn->startTime = std::chrono::steady_clock::now();
+
+    return conn;
+}
+
+
+void RemoteStore::initConnection(Connection & conn)
+{
+    /* Send the magic greeting, check for the reply. */
+    try {
+        conn.to << WORKER_MAGIC_1;
+        conn.to.flush();
+        unsigned int magic = readInt(conn.from);
+        if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
+
+        conn.from >> conn.daemonVersion;
+        if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
+            throw Error("Nix daemon protocol version not supported");
+        if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
+            throw Error("the Nix daemon version is too old");
+        conn.to << PROTOCOL_VERSION;
+
+        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
+            int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1;
+            if (cpu != -1)
+                conn.to << 1 << cpu;
+            else
+                conn.to << 0;
+        }
+
+        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
+            conn.to << false;
+
+        auto ex = conn.processStderr();
+        if (ex) std::rethrow_exception(ex);
+    }
+    catch (Error & e) {
+        throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
+    }
+
+    setOptions(conn);
+}
+
+
+void RemoteStore::setOptions(Connection & conn)
+{
+    conn.to << wopSetOptions
+       << settings.keepFailed
+       << settings.keepGoing
+       << settings.tryFallback
+       << verbosity
+       << settings.maxBuildJobs
+       << settings.maxSilentTime
+       << true
+       << (settings.verboseBuild ? lvlError : lvlVomit)
+       << 0 // obsolete log type
+       << 0 /* obsolete print build trace */
+       << settings.buildCores
+       << settings.useSubstitutes;
+
+    if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
+        std::map<std::string, Config::SettingInfo> overrides;
+        globalConfig.getSettings(overrides, true);
+        overrides.erase(settings.keepFailed.name);
+        overrides.erase(settings.keepGoing.name);
+        overrides.erase(settings.tryFallback.name);
+        overrides.erase(settings.maxBuildJobs.name);
+        overrides.erase(settings.maxSilentTime.name);
+        overrides.erase(settings.buildCores.name);
+        overrides.erase(settings.useSubstitutes.name);
+        overrides.erase(loggerSettings.showTrace.name);
+        conn.to << overrides.size();
+        for (auto & i : overrides)
+            conn.to << i.first << i.second.value;
+    }
+
+    auto ex = conn.processStderr();
+    if (ex) std::rethrow_exception(ex);
+}
+
+
+/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
+   the connection as bad (causing it to be closed) if a non-daemon
+   exception is thrown before the handle is closed. Such an exception
+   causes a deviation from the expected protocol and therefore a
+   desynchronization between the client and daemon. */
+struct ConnectionHandle
+{
+    Pool<RemoteStore::Connection>::Handle handle;
+    bool daemonException = false;
+
+    ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
+        : handle(std::move(handle))
+    { }
+
+    ConnectionHandle(ConnectionHandle && h)
+        : handle(std::move(h.handle))
+    { }
+
+    ~ConnectionHandle()
+    {
+        if (!daemonException && std::uncaught_exceptions()) {
+            handle.markBad();
+            debug("closing daemon connection because of an exception");
+        }
+    }
+
+    RemoteStore::Connection * operator -> () { return &*handle; }
+
+    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
+    {
+        auto ex = handle->processStderr(sink, source, flush);
+        if (ex) {
+            daemonException = true;
+            std::rethrow_exception(ex);
+        }
+    }
+
+    void withFramedSink(std::function<void(Sink & sink)> fun);
+};
+
+
+ConnectionHandle RemoteStore::getConnection()
+{
+    return ConnectionHandle(connections->get());
+}
+
+
+bool RemoteStore::isValidPathUncached(const StorePath & path)
+{
+    auto conn(getConnection());
+    conn->to << wopIsValidPath << printStorePath(path);
+    conn.processStderr();
+    return readInt(conn->from);
+}
+
+
+StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
+{
+    auto conn(getConnection());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
+        StorePathSet res;
+        for (auto & i : paths)
+            if (isValidPath(i)) res.insert(i);
+        return res;
+    } else {
+        conn->to << wopQueryValidPaths;
+        worker_proto::write(*this, conn->to, paths);
+        conn.processStderr();
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    }
+}
+
+
+StorePathSet RemoteStore::queryAllValidPaths()
+{
+    auto conn(getConnection());
+    conn->to << wopQueryAllValidPaths;
+    conn.processStderr();
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+}
+
+
+StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
+{
+    auto conn(getConnection());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
+        StorePathSet res;
+        for (auto & i : paths) {
+            conn->to << wopHasSubstitutes << printStorePath(i);
+            conn.processStderr();
+            if (readInt(conn->from)) res.insert(i);
+        }
+        return res;
+    } else {
+        conn->to << wopQuerySubstitutablePaths;
+        worker_proto::write(*this, conn->to, paths);
+        conn.processStderr();
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    }
+}
+
+
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
+{
+    if (pathsMap.empty()) return;
+
+    auto conn(getConnection());
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
+
+        for (auto & i : pathsMap) {
+            SubstitutablePathInfo info;
+            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
+            conn.processStderr();
+            unsigned int reply = readInt(conn->from);
+            if (reply == 0) continue;
+            auto deriver = readString(conn->from);
+            if (deriver != "")
+                info.deriver = parseStorePath(deriver);
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.downloadSize = readLongLong(conn->from);
+            info.narSize = readLongLong(conn->from);
+            infos.insert_or_assign(i.first, std::move(info));
+        }
+
+    } else {
+
+        conn->to << wopQuerySubstitutablePathInfos;
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+            StorePathSet paths;
+            for (auto & path : pathsMap)
+                paths.insert(path.first);
+            worker_proto::write(*this, conn->to, paths);
+        } else
+            worker_proto::write(*this, conn->to, pathsMap);
+        conn.processStderr();
+        size_t count = readNum<size_t>(conn->from);
+        for (size_t n = 0; n < count; n++) {
+            SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
+            auto deriver = readString(conn->from);
+            if (deriver != "")
+                info.deriver = parseStorePath(deriver);
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.downloadSize = readLongLong(conn->from);
+            info.narSize = readLongLong(conn->from);
+        }
+
+    }
+}
+
+
+ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
+{
+    auto deriver = readString(conn->from);
+    auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
+    auto info = make_ref<ValidPathInfo>(path, narHash);
+    if (deriver != "") info->deriver = parseStorePath(deriver);
+    info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    conn->from >> info->registrationTime >> info->narSize;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
+        conn->from >> info->ultimate;
+        info->sigs = readStrings<StringSet>(conn->from);
+        info->ca = parseContentAddressOpt(readString(conn->from));
+    }
+    return info;
+}
+
+
+void RemoteStore::queryPathInfoUncached(const StorePath & path,
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
+{
+    try {
+        std::shared_ptr<const ValidPathInfo> info;
+        {
+            auto conn(getConnection());
+            conn->to << wopQueryPathInfo << printStorePath(path);
+            try {
+                conn.processStderr();
+            } catch (Error & e) {
+                // Ugly backwards compatibility hack.
+                if (e.msg().find("is not valid") != std::string::npos)
+                    throw InvalidPath(e.info());
+                throw;
+            }
+            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
+                bool valid; conn->from >> valid;
+                if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
+            }
+            info = readValidPathInfo(conn, path);
+        }
+        callback(std::move(info));
+    } catch (...) { callback.rethrow(); }
+}
+
+
+void RemoteStore::queryReferrers(const StorePath & path,
+    StorePathSet & referrers)
+{
+    auto conn(getConnection());
+    conn->to << wopQueryReferrers << printStorePath(path);
+    conn.processStderr();
+    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+        referrers.insert(i);
+}
+
+
+StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
+{
+    auto conn(getConnection());
+    conn->to << wopQueryValidDerivers << printStorePath(path);
+    conn.processStderr();
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+}
+
+
+StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
+{
+    auto conn(getConnection());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
+        return Store::queryDerivationOutputs(path);
+    }
+    conn->to << wopQueryDerivationOutputs << printStorePath(path);
+    conn.processStderr();
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+}
+
+
+std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
+{
+    if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
+        auto conn(getConnection());
+        conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+        conn.processStderr();
+        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+    } else {
+        // Fallback for old daemon versions.
+        // For floating-CA derivations (and their co-dependencies) this is an
+        // under-approximation as it only returns the paths that can be inferred
+        // from the derivation itself (and not the ones that are known because
+        // the have been built), but as old stores don't handle floating-CA
+        // derivations this shouldn't matter
+        auto derivation = readDerivation(path);
+        auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
+        std::map<std::string, std::optional<StorePath>> ret;
+        for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
+            ret.emplace(outputName, outputAndPath.second);
+        }
+        return ret;
+    }
+}
+
+std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
+{
+    auto conn(getConnection());
+    conn->to << wopQueryPathFromHashPart << hashPart;
+    conn.processStderr();
+    Path path = readString(conn->from);
+    if (path.empty()) return {};
+    return parseStorePath(path);
+}
+
+
+ref<const ValidPathInfo> RemoteStore::addCAToStore(
+    Source & dump,
+    const string & name,
+    ContentAddressMethod caMethod,
+    const StorePathSet & references,
+    RepairFlag repair)
+{
+    std::optional<ConnectionHandle> conn_(getConnection());
+    auto & conn = *conn_;
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
+
+        conn->to
+            << wopAddToStore
+            << name
+            << renderContentAddressMethod(caMethod);
+        worker_proto::write(*this, conn->to, references);
+        conn->to << repair;
+
+        conn.withFramedSink([&](Sink & sink) {
+            dump.drainInto(sink);
+        });
+
+        auto path = parseStorePath(readString(conn->from));
+        return readValidPathInfo(conn, path);
+    }
+    else {
+        if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
+
+        std::visit(overloaded {
+            [&](TextHashMethod thm) -> void {
+                std::string s = dump.drain();
+                conn->to << wopAddTextToStore << name << s;
+                worker_proto::write(*this, conn->to, references);
+                conn.processStderr();
+            },
+            [&](FixedOutputHashMethod fohm) -> void {
+                conn->to
+                    << wopAddToStore
+                    << name
+                    << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
+                    << printHashType(fohm.hashType);
+
+                try {
+                    conn->to.written = 0;
+                    conn->to.warn = true;
+                    connections->incCapacity();
+                    {
+                        Finally cleanup([&]() { connections->decCapacity(); });
+                        if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
+                            dump.drainInto(conn->to);
+                        } else {
+                            std::string contents = dump.drain();
+                            dumpString(contents, conn->to);
+                        }
+                    }
+                    conn->to.warn = false;
+                    conn.processStderr();
+                } catch (SysError & e) {
+                    /* Daemon closed while we were sending the path. Probably OOM
+                      or I/O error. */
+                    if (e.errNo == EPIPE)
+                        try {
+                            conn.processStderr();
+                        } catch (EndOfFile & e) { }
+                    throw;
+                }
+
+            }
+        }, caMethod);
+        auto path = parseStorePath(readString(conn->from));
+        // Release our connection to prevent a deadlock in queryPathInfo().
+        conn_.reset();
+        return queryPathInfo(path);
+    }
+}
+
+
+StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashType, RepairFlag repair)
+{
+    StorePathSet references;
+    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
+}
+
+
+void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    auto conn(getConnection());
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
+        conn->to << wopImportPaths;
+
+        auto source2 = sinkToSource([&](Sink & sink) {
+            sink << 1 // == path follows
+                ;
+            copyNAR(source, sink);
+            sink
+                << exportMagic
+                << printStorePath(info.path);
+            worker_proto::write(*this, sink, info.references);
+            sink
+                << (info.deriver ? printStorePath(*info.deriver) : "")
+                << 0 // == no legacy signature
+                << 0 // == no path follows
+                ;
+        });
+
+        conn.processStderr(0, source2.get());
+
+        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        assert(importedPaths.size() <= 1);
+    }
+
+    else {
+        conn->to << wopAddToStoreNar
+                 << printStorePath(info.path)
+                 << (info.deriver ? printStorePath(*info.deriver) : "")
+                 << info.narHash.to_string(Base16, false);
+        worker_proto::write(*this, conn->to, info.references);
+        conn->to << info.registrationTime << info.narSize
+                 << info.ultimate << info.sigs << renderContentAddress(info.ca)
+                 << repair << !checkSigs;
+
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
+            conn.withFramedSink([&](Sink & sink) {
+                copyNAR(source, sink);
+            });
+        } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
+            conn.processStderr(0, &source);
+        } else {
+            copyNAR(source, conn->to);
+            conn.processStderr(0, nullptr);
+        }
+    }
+}
+
+
+StorePath RemoteStore::addTextToStore(const string & name, const string & s,
+    const StorePathSet & references, RepairFlag repair)
+{
+    StringSource source(s);
+    return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
+}
+
+
+void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+{
+    auto conn(getConnection());
+    conn->to << wopBuildPaths;
+    assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
+    Strings ss;
+    for (auto & p : drvPaths)
+        ss.push_back(p.to_string(*this));
+    conn->to << ss;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
+        conn->to << buildMode;
+    else
+        /* Old daemons did not take a 'buildMode' parameter, so we
+           need to validate it here on the client side.  */
+        if (buildMode != bmNormal)
+            throw Error("repairing or checking is not supported when building through the Nix daemon");
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+    BuildMode buildMode)
+{
+    auto conn(getConnection());
+    conn->to << wopBuildDerivation << printStorePath(drvPath);
+    writeDerivation(conn->to, *this, drv);
+    conn->to << buildMode;
+    conn.processStderr();
+    BuildResult res;
+    unsigned int status;
+    conn->from >> status >> res.errorMsg;
+    res.status = (BuildResult::Status) status;
+    return res;
+}
+
+
+void RemoteStore::ensurePath(const StorePath & path)
+{
+    auto conn(getConnection());
+    conn->to << wopEnsurePath << printStorePath(path);
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+void RemoteStore::addTempRoot(const StorePath & path)
+{
+    auto conn(getConnection());
+    conn->to << wopAddTempRoot << printStorePath(path);
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+void RemoteStore::addIndirectRoot(const Path & path)
+{
+    auto conn(getConnection());
+    conn->to << wopAddIndirectRoot << path;
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+void RemoteStore::syncWithGC()
+{
+    auto conn(getConnection());
+    conn->to << wopSyncWithGC;
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+Roots RemoteStore::findRoots(bool censor)
+{
+    auto conn(getConnection());
+    conn->to << wopFindRoots;
+    conn.processStderr();
+    size_t count = readNum<size_t>(conn->from);
+    Roots result;
+    while (count--) {
+        Path link = readString(conn->from);
+        auto target = parseStorePath(readString(conn->from));
+        result[std::move(target)].emplace(link);
+    }
+    return result;
+}
+
+
+void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
+    auto conn(getConnection());
+
+    conn->to
+        << wopCollectGarbage << options.action;
+    worker_proto::write(*this, conn->to, options.pathsToDelete);
+    conn->to << options.ignoreLiveness
+        << options.maxFreed
+        /* removed options */
+        << 0 << 0 << 0;
+
+    conn.processStderr();
+
+    results.paths = readStrings<PathSet>(conn->from);
+    results.bytesFreed = readLongLong(conn->from);
+    readLongLong(conn->from); // obsolete
+
+    {
+        auto state_(Store::state.lock());
+        state_->pathInfoCache.clear();
+    }
+}
+
+
+void RemoteStore::optimiseStore()
+{
+    auto conn(getConnection());
+    conn->to << wopOptimiseStore;
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
+{
+    auto conn(getConnection());
+    conn->to << wopVerifyStore << checkContents << repair;
+    conn.processStderr();
+    return readInt(conn->from);
+}
+
+
+void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
+{
+    auto conn(getConnection());
+    conn->to << wopAddSignatures << printStorePath(storePath) << sigs;
+    conn.processStderr();
+    readInt(conn->from);
+}
+
+
+void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+    StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
+    uint64_t & downloadSize, uint64_t & narSize)
+{
+    {
+        auto conn(getConnection());
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
+            // Don't hold the connection handle in the fallback case
+            // to prevent a deadlock.
+            goto fallback;
+        conn->to << wopQueryMissing;
+        Strings ss;
+        for (auto & p : targets)
+            ss.push_back(p.to_string(*this));
+        conn->to << ss;
+        conn.processStderr();
+        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        conn->from >> downloadSize >> narSize;
+        return;
+    }
+
+ fallback:
+    return Store::queryMissing(targets, willBuild, willSubstitute,
+        unknown, downloadSize, narSize);
+}
+
+
+void RemoteStore::connect()
+{
+    auto conn(getConnection());
+}
+
+
+unsigned int RemoteStore::getProtocol()
+{
+    auto conn(connections->get());
+    return conn->daemonVersion;
+}
+
+
+void RemoteStore::flushBadConnections()
+{
+    connections->flushBad();
+}
+
+
+RemoteStore::Connection::~Connection()
+{
+    try {
+        to.flush();
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
+{
+    auto conn(connections->get());
+    conn->to << wopNarFromPath << printStorePath(path);
+    conn->processStderr();
+    copyNAR(conn->from, sink);
+}
+
+ref<FSAccessor> RemoteStore::getFSAccessor()
+{
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
+}
+
+static Logger::Fields readFields(Source & from)
+{
+    Logger::Fields fields;
+    size_t size = readInt(from);
+    for (size_t n = 0; n < size; n++) {
+        auto type = (decltype(Logger::Field::type)) readInt(from);
+        if (type == Logger::Field::tInt)
+            fields.push_back(readNum<uint64_t>(from));
+        else if (type == Logger::Field::tString)
+            fields.push_back(readString(from));
+        else
+            throw Error("got unsupported field type %x from Nix daemon", (int) type);
+    }
+    return fields;
+}
+
+
+std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
+{
+    if (flush)
+        to.flush();
+
+    while (true) {
+
+        auto msg = readNum<uint64_t>(from);
+
+        if (msg == STDERR_WRITE) {
+            string s = readString(from);
+            if (!sink) throw Error("no sink");
+            (*sink)(s);
+        }
+
+        else if (msg == STDERR_READ) {
+            if (!source) throw Error("no source");
+            size_t len = readNum<size_t>(from);
+            auto buf = std::make_unique<unsigned char[]>(len);
+            writeString(buf.get(), source->read(buf.get(), len), to);
+            to.flush();
+        }
+
+        else if (msg == STDERR_ERROR) {
+            if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
+                return std::make_exception_ptr(readError(from));
+            } else {
+                string error = readString(from);
+                unsigned int status = readInt(from);
+                return std::make_exception_ptr(Error(status, error));
+            }
+        }
+
+        else if (msg == STDERR_NEXT)
+            printError(chomp(readString(from)));
+
+        else if (msg == STDERR_START_ACTIVITY) {
+            auto act = readNum<ActivityId>(from);
+            auto lvl = (Verbosity) readInt(from);
+            auto type = (ActivityType) readInt(from);
+            auto s = readString(from);
+            auto fields = readFields(from);
+            auto parent = readNum<ActivityId>(from);
+            logger->startActivity(act, lvl, type, s, fields, parent);
+        }
+
+        else if (msg == STDERR_STOP_ACTIVITY) {
+            auto act = readNum<ActivityId>(from);
+            logger->stopActivity(act);
+        }
+
+        else if (msg == STDERR_RESULT) {
+            auto act = readNum<ActivityId>(from);
+            auto type = (ResultType) readInt(from);
+            auto fields = readFields(from);
+            logger->result(act, type, fields);
+        }
+
+        else if (msg == STDERR_LAST)
+            break;
+
+        else
+            throw Error("got unknown message type %x from Nix daemon", msg);
+    }
+
+    return nullptr;
+}
+
+void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
+{
+    (*this)->to.flush();
+
+    std::exception_ptr ex;
+
+    /* Handle log messages / exceptions from the remote on a
+        separate thread. */
+    std::thread stderrThread([&]()
+    {
+        try {
+            processStderr(nullptr, nullptr, false);
+        } catch (...) {
+            ex = std::current_exception();
+        }
+    });
+
+    Finally joinStderrThread([&]()
+    {
+        if (stderrThread.joinable()) {
+            stderrThread.join();
+            if (ex) {
+                try {
+                    std::rethrow_exception(ex);
+                } catch (...) {
+                    ignoreException();
+                }
+            }
+        }
+    });
+
+    {
+        FramedSink sink((*this)->to, ex);
+        fun(sink);
+        sink.flush();
+    }
+
+    stderrThread.join();
+    if (ex)
+        std::rethrow_exception(ex);
+
+}
+
+static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
+
+}
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
new file mode 100644
index 000000000..554fb6bed
--- /dev/null
+++ b/src/libstore/uds-remote-store.hh
@@ -0,0 +1,204 @@
+#pragma once
+
+#include <limits>
+#include <string>
+
+#include "store-api.hh"
+#include "local-fs-store.hh"
+
+
+namespace nix {
+
+
+class Pipe;
+class Pid;
+struct FdSink;
+struct FdSource;
+template<typename T> class Pool;
+struct ConnectionHandle;
+
+struct RemoteStoreConfig : virtual StoreConfig
+{
+    using StoreConfig::StoreConfig;
+
+    const Setting<int> maxConnections{(StoreConfig*) this, 1,
+            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
+
+    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
+            "max-connection-age", "number of seconds to reuse a connection"};
+};
+
+/* FIXME: RemoteStore is a misnomer - should be something like
+   DaemonStore. */
+class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
+{
+public:
+
+    virtual bool sameMachine() = 0;
+
+    RemoteStore(const Params & params);
+
+    /* Implementations of abstract store API methods. */
+
+    bool isValidPathUncached(const StorePath & path) override;
+
+    StorePathSet queryValidPaths(const StorePathSet & paths,
+        SubstituteFlag maybeSubstitute = NoSubstitute) override;
+
+    StorePathSet queryAllValidPaths() override;
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
+
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
+
+    StorePathSet queryValidDerivers(const StorePath & path) override;
+
+    StorePathSet queryDerivationOutputs(const StorePath & path) override;
+
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
+
+    StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
+
+    void querySubstitutablePathInfos(const StorePathCAMap & paths,
+        SubstitutablePathInfos & infos) override;
+
+    /* Add a content-addressable store path. `dump` will be drained. */
+    ref<const ValidPathInfo> addCAToStore(
+        Source & dump,
+        const string & name,
+        ContentAddressMethod caMethod,
+        const StorePathSet & references,
+        RepairFlag repair);
+
+    /* Add a content-addressable store path. Does not support references. `dump` will be drained. */
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+
+    void addToStore(const ValidPathInfo & info, Source & nar,
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
+
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override;
+
+    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override;
+
+    void ensurePath(const StorePath & path) override;
+
+    void addTempRoot(const StorePath & path) override;
+
+    void addIndirectRoot(const Path & path) override;
+
+    void syncWithGC() override;
+
+    Roots findRoots(bool censor) override;
+
+    void collectGarbage(const GCOptions & options, GCResults & results) override;
+
+    void optimiseStore() override;
+
+    bool verifyStore(bool checkContents, RepairFlag repair) override;
+
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
+
+    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
+        uint64_t & downloadSize, uint64_t & narSize) override;
+
+    void connect() override;
+
+    unsigned int getProtocol() override;
+
+    void flushBadConnections();
+
+    struct Connection
+    {
+        AutoCloseFD fd;
+        FdSink to;
+        FdSource from;
+        unsigned int daemonVersion;
+        std::chrono::time_point<std::chrono::steady_clock> startTime;
+
+        virtual ~Connection();
+
+        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
+    };
+
+    ref<Connection> openConnectionWrapper();
+
+protected:
+
+    virtual ref<Connection> openConnection() = 0;
+
+    void initConnection(Connection & conn);
+
+    ref<Pool<Connection>> connections;
+
+    virtual void setOptions(Connection & conn);
+
+    ConnectionHandle getConnection();
+
+    friend struct ConnectionHandle;
+
+    virtual ref<FSAccessor> getFSAccessor() override;
+
+    virtual void narFromPath(const StorePath & path, Sink & sink) override;
+
+    ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
+
+private:
+
+    std::atomic_bool failed{false};
+
+};
+
+struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
+{
+    UDSRemoteStoreConfig(const Store::Params & params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , RemoteStoreConfig(params)
+    {
+    }
+
+    UDSRemoteStoreConfig()
+        : UDSRemoteStoreConfig(Store::Params({}))
+    {
+    }
+
+    const std::string name() override { return "Local Daemon Store"; }
+};
+
+class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
+{
+public:
+
+    UDSRemoteStore(const Params & params);
+    UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
+
+    std::string getUri() override;
+
+    static std::set<std::string> uriSchemes()
+    { return {"unix"}; }
+
+    bool sameMachine() override
+    { return true; }
+
+    ref<FSAccessor> getFSAccessor() override
+    { return LocalFSStore::getFSAccessor(); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { LocalFSStore::narFromPath(path, sink); }
+
+private:
+
+    ref<RemoteStore::Connection> openConnection() override;
+    std::optional<std::string> path;
+};
+
+
+}

From 15fdb7cc6bb9e86283a67cce646981821dca1558 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 11 Oct 2020 16:01:53 +0000
Subject: [PATCH 375/882] Split out uds-remote-store.{cc.hh}

---
 src/libstore/remote-store.cc     |  75 ---
 src/libstore/remote-store.hh     |  45 --
 src/libstore/store-api.cc        |   2 +-
 src/libstore/uds-remote-store.cc | 940 +------------------------------
 src/libstore/uds-remote-store.hh | 154 +----
 5 files changed, 4 insertions(+), 1212 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 23b1942ce..488270f48 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -12,16 +12,6 @@
 #include "logging.hh"
 #include "callback.hh"
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <cstring>
-
 namespace nix {
 
 namespace worker_proto {
@@ -125,69 +115,6 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 }
 
 
-UDSRemoteStore::UDSRemoteStore(const Params & params)
-    : StoreConfig(params)
-    , Store(params)
-    , LocalFSStore(params)
-    , RemoteStore(params)
-{
-}
-
-
-UDSRemoteStore::UDSRemoteStore(
-        const std::string scheme,
-        std::string socket_path,
-        const Params & params)
-    : UDSRemoteStore(params)
-{
-    path.emplace(socket_path);
-}
-
-
-std::string UDSRemoteStore::getUri()
-{
-    if (path) {
-        return std::string("unix://") + *path;
-    } else {
-        return "daemon";
-    }
-}
-
-
-ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
-{
-    auto conn = make_ref<Connection>();
-
-    /* Connect to a daemon that does the privileged work for us. */
-    conn->fd = socket(PF_UNIX, SOCK_STREAM
-        #ifdef SOCK_CLOEXEC
-        | SOCK_CLOEXEC
-        #endif
-        , 0);
-    if (!conn->fd)
-        throw SysError("cannot create Unix domain socket");
-    closeOnExec(conn->fd.get());
-
-    string socketPath = path ? *path : settings.nixDaemonSocketFile;
-
-    struct sockaddr_un addr;
-    addr.sun_family = AF_UNIX;
-    if (socketPath.size() + 1 >= sizeof(addr.sun_path))
-        throw Error("socket path '%1%' is too long", socketPath);
-    strcpy(addr.sun_path, socketPath.c_str());
-
-    if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
-        throw SysError("cannot connect to daemon at '%1%'", socketPath);
-
-    conn->from.fd = conn->fd.get();
-    conn->to.fd = conn->fd.get();
-
-    conn->startTime = std::chrono::steady_clock::now();
-
-    return conn;
-}
-
-
 void RemoteStore::initConnection(Connection & conn)
 {
     /* Send the magic greeting, check for the reply. */
@@ -1012,6 +939,4 @@ void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
 
 }
 
-static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
-
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 554fb6bed..9f78fcb02 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -4,7 +4,6 @@
 #include <string>
 
 #include "store-api.hh"
-#include "local-fs-store.hh"
 
 
 namespace nix {
@@ -156,49 +155,5 @@ private:
 
 };
 
-struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
-{
-    UDSRemoteStoreConfig(const Store::Params & params)
-        : StoreConfig(params)
-        , LocalFSStoreConfig(params)
-        , RemoteStoreConfig(params)
-    {
-    }
-
-    UDSRemoteStoreConfig()
-        : UDSRemoteStoreConfig(Store::Params({}))
-    {
-    }
-
-    const std::string name() override { return "Local Daemon Store"; }
-};
-
-class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
-{
-public:
-
-    UDSRemoteStore(const Params & params);
-    UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
-
-    std::string getUri() override;
-
-    static std::set<std::string> uriSchemes()
-    { return {"unix"}; }
-
-    bool sameMachine() override
-    { return true; }
-
-    ref<FSAccessor> getFSAccessor() override
-    { return LocalFSStore::getFSAccessor(); }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    { LocalFSStore::narFromPath(path, sink); }
-
-private:
-
-    ref<RemoteStore::Connection> openConnection() override;
-    std::optional<std::string> path;
-};
-
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1bbc74db8..9f21f0434 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1011,7 +1011,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
 
 
 #include "local-store.hh"
-#include "remote-store.hh"
+#include "uds-remote-store.hh"
 
 
 namespace nix {
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index 23b1942ce..24f3e9c6d 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -1,16 +1,4 @@
-#include "serialise.hh"
-#include "util.hh"
-#include "remote-fs-accessor.hh"
-#include "remote-store.hh"
-#include "worker-protocol.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "globals.hh"
-#include "derivations.hh"
-#include "pool.hh"
-#include "finally.hh"
-#include "logging.hh"
-#include "callback.hh"
+#include "uds-remote-store.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -22,109 +10,9 @@
 
 #include <cstring>
 
+
 namespace nix {
 
-namespace worker_proto {
-
-std::string read(const Store & store, Source & from, Phantom<std::string> _)
-{
-    return readString(from);
-}
-
-void write(const Store & store, Sink & out, const std::string & str)
-{
-    out << str;
-}
-
-
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
-{
-    return store.parseStorePath(readString(from));
-}
-
-void write(const Store & store, Sink & out, const StorePath & storePath)
-{
-    out << store.printStorePath(storePath);
-}
-
-
-ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
-{
-    return parseContentAddress(readString(from));
-}
-
-void write(const Store & store, Sink & out, const ContentAddress & ca)
-{
-    out << renderContentAddress(ca);
-}
-
-
-std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
-{
-    auto s = readString(from);
-    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
-}
-
-void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
-{
-    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
-}
-
-
-std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
-{
-    return parseContentAddressOpt(readString(from));
-}
-
-void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
-{
-    out << (caOpt ? renderContentAddress(*caOpt) : "");
-}
-
-}
-
-
-/* TODO: Separate these store impls into different files, give them better names */
-RemoteStore::RemoteStore(const Params & params)
-    : Store(params)
-    , RemoteStoreConfig(params)
-    , connections(make_ref<Pool<Connection>>(
-            std::max(1, (int) maxConnections),
-            [this]() {
-                auto conn = openConnectionWrapper();
-                try {
-                    initConnection(*conn);
-                } catch (...) {
-                    failed = true;
-                    throw;
-                }
-                return conn;
-            },
-            [this](const ref<Connection> & r) {
-                return
-                    r->to.good()
-                    && r->from.good()
-                    && std::chrono::duration_cast<std::chrono::seconds>(
-                        std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
-            }
-            ))
-{
-}
-
-
-ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
-{
-    if (failed)
-        throw Error("opening a connection to remote store '%s' previously failed", getUri());
-    try {
-        return openConnection();
-    } catch (...) {
-        failed = true;
-        throw;
-    }
-}
-
-
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : StoreConfig(params)
     , Store(params)
@@ -188,830 +76,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
 }
 
 
-void RemoteStore::initConnection(Connection & conn)
-{
-    /* Send the magic greeting, check for the reply. */
-    try {
-        conn.to << WORKER_MAGIC_1;
-        conn.to.flush();
-        unsigned int magic = readInt(conn.from);
-        if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
-
-        conn.from >> conn.daemonVersion;
-        if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
-            throw Error("Nix daemon protocol version not supported");
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
-            throw Error("the Nix daemon version is too old");
-        conn.to << PROTOCOL_VERSION;
-
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
-            int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1;
-            if (cpu != -1)
-                conn.to << 1 << cpu;
-            else
-                conn.to << 0;
-        }
-
-        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
-            conn.to << false;
-
-        auto ex = conn.processStderr();
-        if (ex) std::rethrow_exception(ex);
-    }
-    catch (Error & e) {
-        throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
-    }
-
-    setOptions(conn);
-}
-
-
-void RemoteStore::setOptions(Connection & conn)
-{
-    conn.to << wopSetOptions
-       << settings.keepFailed
-       << settings.keepGoing
-       << settings.tryFallback
-       << verbosity
-       << settings.maxBuildJobs
-       << settings.maxSilentTime
-       << true
-       << (settings.verboseBuild ? lvlError : lvlVomit)
-       << 0 // obsolete log type
-       << 0 /* obsolete print build trace */
-       << settings.buildCores
-       << settings.useSubstitutes;
-
-    if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
-        std::map<std::string, Config::SettingInfo> overrides;
-        globalConfig.getSettings(overrides, true);
-        overrides.erase(settings.keepFailed.name);
-        overrides.erase(settings.keepGoing.name);
-        overrides.erase(settings.tryFallback.name);
-        overrides.erase(settings.maxBuildJobs.name);
-        overrides.erase(settings.maxSilentTime.name);
-        overrides.erase(settings.buildCores.name);
-        overrides.erase(settings.useSubstitutes.name);
-        overrides.erase(loggerSettings.showTrace.name);
-        conn.to << overrides.size();
-        for (auto & i : overrides)
-            conn.to << i.first << i.second.value;
-    }
-
-    auto ex = conn.processStderr();
-    if (ex) std::rethrow_exception(ex);
-}
-
-
-/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
-   the connection as bad (causing it to be closed) if a non-daemon
-   exception is thrown before the handle is closed. Such an exception
-   causes a deviation from the expected protocol and therefore a
-   desynchronization between the client and daemon. */
-struct ConnectionHandle
-{
-    Pool<RemoteStore::Connection>::Handle handle;
-    bool daemonException = false;
-
-    ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
-        : handle(std::move(handle))
-    { }
-
-    ConnectionHandle(ConnectionHandle && h)
-        : handle(std::move(h.handle))
-    { }
-
-    ~ConnectionHandle()
-    {
-        if (!daemonException && std::uncaught_exceptions()) {
-            handle.markBad();
-            debug("closing daemon connection because of an exception");
-        }
-    }
-
-    RemoteStore::Connection * operator -> () { return &*handle; }
-
-    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
-    {
-        auto ex = handle->processStderr(sink, source, flush);
-        if (ex) {
-            daemonException = true;
-            std::rethrow_exception(ex);
-        }
-    }
-
-    void withFramedSink(std::function<void(Sink & sink)> fun);
-};
-
-
-ConnectionHandle RemoteStore::getConnection()
-{
-    return ConnectionHandle(connections->get());
-}
-
-
-bool RemoteStore::isValidPathUncached(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopIsValidPath << printStorePath(path);
-    conn.processStderr();
-    return readInt(conn->from);
-}
-
-
-StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
-{
-    auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
-        StorePathSet res;
-        for (auto & i : paths)
-            if (isValidPath(i)) res.insert(i);
-        return res;
-    } else {
-        conn->to << wopQueryValidPaths;
-        worker_proto::write(*this, conn->to, paths);
-        conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-    }
-}
-
-
-StorePathSet RemoteStore::queryAllValidPaths()
-{
-    auto conn(getConnection());
-    conn->to << wopQueryAllValidPaths;
-    conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-}
-
-
-StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
-{
-    auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
-        StorePathSet res;
-        for (auto & i : paths) {
-            conn->to << wopHasSubstitutes << printStorePath(i);
-            conn.processStderr();
-            if (readInt(conn->from)) res.insert(i);
-        }
-        return res;
-    } else {
-        conn->to << wopQuerySubstitutablePaths;
-        worker_proto::write(*this, conn->to, paths);
-        conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-    }
-}
-
-
-void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
-{
-    if (pathsMap.empty()) return;
-
-    auto conn(getConnection());
-
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
-
-        for (auto & i : pathsMap) {
-            SubstitutablePathInfo info;
-            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
-            conn.processStderr();
-            unsigned int reply = readInt(conn->from);
-            if (reply == 0) continue;
-            auto deriver = readString(conn->from);
-            if (deriver != "")
-                info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-            info.downloadSize = readLongLong(conn->from);
-            info.narSize = readLongLong(conn->from);
-            infos.insert_or_assign(i.first, std::move(info));
-        }
-
-    } else {
-
-        conn->to << wopQuerySubstitutablePathInfos;
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
-            StorePathSet paths;
-            for (auto & path : pathsMap)
-                paths.insert(path.first);
-            worker_proto::write(*this, conn->to, paths);
-        } else
-            worker_proto::write(*this, conn->to, pathsMap);
-        conn.processStderr();
-        size_t count = readNum<size_t>(conn->from);
-        for (size_t n = 0; n < count; n++) {
-            SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
-            auto deriver = readString(conn->from);
-            if (deriver != "")
-                info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-            info.downloadSize = readLongLong(conn->from);
-            info.narSize = readLongLong(conn->from);
-        }
-
-    }
-}
-
-
-ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
-{
-    auto deriver = readString(conn->from);
-    auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
-    auto info = make_ref<ValidPathInfo>(path, narHash);
-    if (deriver != "") info->deriver = parseStorePath(deriver);
-    info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-    conn->from >> info->registrationTime >> info->narSize;
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-        conn->from >> info->ultimate;
-        info->sigs = readStrings<StringSet>(conn->from);
-        info->ca = parseContentAddressOpt(readString(conn->from));
-    }
-    return info;
-}
-
-
-void RemoteStore::queryPathInfoUncached(const StorePath & path,
-    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
-{
-    try {
-        std::shared_ptr<const ValidPathInfo> info;
-        {
-            auto conn(getConnection());
-            conn->to << wopQueryPathInfo << printStorePath(path);
-            try {
-                conn.processStderr();
-            } catch (Error & e) {
-                // Ugly backwards compatibility hack.
-                if (e.msg().find("is not valid") != std::string::npos)
-                    throw InvalidPath(e.info());
-                throw;
-            }
-            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-                bool valid; conn->from >> valid;
-                if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
-            }
-            info = readValidPathInfo(conn, path);
-        }
-        callback(std::move(info));
-    } catch (...) { callback.rethrow(); }
-}
-
-
-void RemoteStore::queryReferrers(const StorePath & path,
-    StorePathSet & referrers)
-{
-    auto conn(getConnection());
-    conn->to << wopQueryReferrers << printStorePath(path);
-    conn.processStderr();
-    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
-        referrers.insert(i);
-}
-
-
-StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopQueryValidDerivers << printStorePath(path);
-    conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-}
-
-
-StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
-{
-    auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
-        return Store::queryDerivationOutputs(path);
-    }
-    conn->to << wopQueryDerivationOutputs << printStorePath(path);
-    conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-}
-
-
-std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
-{
-    if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
-        auto conn(getConnection());
-        conn->to << wopQueryDerivationOutputMap << printStorePath(path);
-        conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
-    } else {
-        // Fallback for old daemon versions.
-        // For floating-CA derivations (and their co-dependencies) this is an
-        // under-approximation as it only returns the paths that can be inferred
-        // from the derivation itself (and not the ones that are known because
-        // the have been built), but as old stores don't handle floating-CA
-        // derivations this shouldn't matter
-        auto derivation = readDerivation(path);
-        auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
-        std::map<std::string, std::optional<StorePath>> ret;
-        for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
-            ret.emplace(outputName, outputAndPath.second);
-        }
-        return ret;
-    }
-}
-
-std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
-{
-    auto conn(getConnection());
-    conn->to << wopQueryPathFromHashPart << hashPart;
-    conn.processStderr();
-    Path path = readString(conn->from);
-    if (path.empty()) return {};
-    return parseStorePath(path);
-}
-
-
-ref<const ValidPathInfo> RemoteStore::addCAToStore(
-    Source & dump,
-    const string & name,
-    ContentAddressMethod caMethod,
-    const StorePathSet & references,
-    RepairFlag repair)
-{
-    std::optional<ConnectionHandle> conn_(getConnection());
-    auto & conn = *conn_;
-
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
-
-        conn->to
-            << wopAddToStore
-            << name
-            << renderContentAddressMethod(caMethod);
-        worker_proto::write(*this, conn->to, references);
-        conn->to << repair;
-
-        conn.withFramedSink([&](Sink & sink) {
-            dump.drainInto(sink);
-        });
-
-        auto path = parseStorePath(readString(conn->from));
-        return readValidPathInfo(conn, path);
-    }
-    else {
-        if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
-
-        std::visit(overloaded {
-            [&](TextHashMethod thm) -> void {
-                std::string s = dump.drain();
-                conn->to << wopAddTextToStore << name << s;
-                worker_proto::write(*this, conn->to, references);
-                conn.processStderr();
-            },
-            [&](FixedOutputHashMethod fohm) -> void {
-                conn->to
-                    << wopAddToStore
-                    << name
-                    << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-                    << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
-                    << printHashType(fohm.hashType);
-
-                try {
-                    conn->to.written = 0;
-                    conn->to.warn = true;
-                    connections->incCapacity();
-                    {
-                        Finally cleanup([&]() { connections->decCapacity(); });
-                        if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
-                            dump.drainInto(conn->to);
-                        } else {
-                            std::string contents = dump.drain();
-                            dumpString(contents, conn->to);
-                        }
-                    }
-                    conn->to.warn = false;
-                    conn.processStderr();
-                } catch (SysError & e) {
-                    /* Daemon closed while we were sending the path. Probably OOM
-                      or I/O error. */
-                    if (e.errNo == EPIPE)
-                        try {
-                            conn.processStderr();
-                        } catch (EndOfFile & e) { }
-                    throw;
-                }
-
-            }
-        }, caMethod);
-        auto path = parseStorePath(readString(conn->from));
-        // Release our connection to prevent a deadlock in queryPathInfo().
-        conn_.reset();
-        return queryPathInfo(path);
-    }
-}
-
-
-StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method, HashType hashType, RepairFlag repair)
-{
-    StorePathSet references;
-    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
-}
-
-
-void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs)
-{
-    auto conn(getConnection());
-
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
-        conn->to << wopImportPaths;
-
-        auto source2 = sinkToSource([&](Sink & sink) {
-            sink << 1 // == path follows
-                ;
-            copyNAR(source, sink);
-            sink
-                << exportMagic
-                << printStorePath(info.path);
-            worker_proto::write(*this, sink, info.references);
-            sink
-                << (info.deriver ? printStorePath(*info.deriver) : "")
-                << 0 // == no legacy signature
-                << 0 // == no path follows
-                ;
-        });
-
-        conn.processStderr(0, source2.get());
-
-        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        assert(importedPaths.size() <= 1);
-    }
-
-    else {
-        conn->to << wopAddToStoreNar
-                 << printStorePath(info.path)
-                 << (info.deriver ? printStorePath(*info.deriver) : "")
-                 << info.narHash.to_string(Base16, false);
-        worker_proto::write(*this, conn->to, info.references);
-        conn->to << info.registrationTime << info.narSize
-                 << info.ultimate << info.sigs << renderContentAddress(info.ca)
-                 << repair << !checkSigs;
-
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
-            conn.withFramedSink([&](Sink & sink) {
-                copyNAR(source, sink);
-            });
-        } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
-            conn.processStderr(0, &source);
-        } else {
-            copyNAR(source, conn->to);
-            conn.processStderr(0, nullptr);
-        }
-    }
-}
-
-
-StorePath RemoteStore::addTextToStore(const string & name, const string & s,
-    const StorePathSet & references, RepairFlag repair)
-{
-    StringSource source(s);
-    return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
-}
-
-
-void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
-{
-    auto conn(getConnection());
-    conn->to << wopBuildPaths;
-    assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
-    Strings ss;
-    for (auto & p : drvPaths)
-        ss.push_back(p.to_string(*this));
-    conn->to << ss;
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
-        conn->to << buildMode;
-    else
-        /* Old daemons did not take a 'buildMode' parameter, so we
-           need to validate it here on the client side.  */
-        if (buildMode != bmNormal)
-            throw Error("repairing or checking is not supported when building through the Nix daemon");
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-    BuildMode buildMode)
-{
-    auto conn(getConnection());
-    conn->to << wopBuildDerivation << printStorePath(drvPath);
-    writeDerivation(conn->to, *this, drv);
-    conn->to << buildMode;
-    conn.processStderr();
-    BuildResult res;
-    unsigned int status;
-    conn->from >> status >> res.errorMsg;
-    res.status = (BuildResult::Status) status;
-    return res;
-}
-
-
-void RemoteStore::ensurePath(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopEnsurePath << printStorePath(path);
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-void RemoteStore::addTempRoot(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopAddTempRoot << printStorePath(path);
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-void RemoteStore::addIndirectRoot(const Path & path)
-{
-    auto conn(getConnection());
-    conn->to << wopAddIndirectRoot << path;
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-void RemoteStore::syncWithGC()
-{
-    auto conn(getConnection());
-    conn->to << wopSyncWithGC;
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-Roots RemoteStore::findRoots(bool censor)
-{
-    auto conn(getConnection());
-    conn->to << wopFindRoots;
-    conn.processStderr();
-    size_t count = readNum<size_t>(conn->from);
-    Roots result;
-    while (count--) {
-        Path link = readString(conn->from);
-        auto target = parseStorePath(readString(conn->from));
-        result[std::move(target)].emplace(link);
-    }
-    return result;
-}
-
-
-void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
-{
-    auto conn(getConnection());
-
-    conn->to
-        << wopCollectGarbage << options.action;
-    worker_proto::write(*this, conn->to, options.pathsToDelete);
-    conn->to << options.ignoreLiveness
-        << options.maxFreed
-        /* removed options */
-        << 0 << 0 << 0;
-
-    conn.processStderr();
-
-    results.paths = readStrings<PathSet>(conn->from);
-    results.bytesFreed = readLongLong(conn->from);
-    readLongLong(conn->from); // obsolete
-
-    {
-        auto state_(Store::state.lock());
-        state_->pathInfoCache.clear();
-    }
-}
-
-
-void RemoteStore::optimiseStore()
-{
-    auto conn(getConnection());
-    conn->to << wopOptimiseStore;
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
-{
-    auto conn(getConnection());
-    conn->to << wopVerifyStore << checkContents << repair;
-    conn.processStderr();
-    return readInt(conn->from);
-}
-
-
-void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
-{
-    auto conn(getConnection());
-    conn->to << wopAddSignatures << printStorePath(storePath) << sigs;
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
-void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
-    StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-    uint64_t & downloadSize, uint64_t & narSize)
-{
-    {
-        auto conn(getConnection());
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
-            // Don't hold the connection handle in the fallback case
-            // to prevent a deadlock.
-            goto fallback;
-        conn->to << wopQueryMissing;
-        Strings ss;
-        for (auto & p : targets)
-            ss.push_back(p.to_string(*this));
-        conn->to << ss;
-        conn.processStderr();
-        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        conn->from >> downloadSize >> narSize;
-        return;
-    }
-
- fallback:
-    return Store::queryMissing(targets, willBuild, willSubstitute,
-        unknown, downloadSize, narSize);
-}
-
-
-void RemoteStore::connect()
-{
-    auto conn(getConnection());
-}
-
-
-unsigned int RemoteStore::getProtocol()
-{
-    auto conn(connections->get());
-    return conn->daemonVersion;
-}
-
-
-void RemoteStore::flushBadConnections()
-{
-    connections->flushBad();
-}
-
-
-RemoteStore::Connection::~Connection()
-{
-    try {
-        to.flush();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
-{
-    auto conn(connections->get());
-    conn->to << wopNarFromPath << printStorePath(path);
-    conn->processStderr();
-    copyNAR(conn->from, sink);
-}
-
-ref<FSAccessor> RemoteStore::getFSAccessor()
-{
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
-}
-
-static Logger::Fields readFields(Source & from)
-{
-    Logger::Fields fields;
-    size_t size = readInt(from);
-    for (size_t n = 0; n < size; n++) {
-        auto type = (decltype(Logger::Field::type)) readInt(from);
-        if (type == Logger::Field::tInt)
-            fields.push_back(readNum<uint64_t>(from));
-        else if (type == Logger::Field::tString)
-            fields.push_back(readString(from));
-        else
-            throw Error("got unsupported field type %x from Nix daemon", (int) type);
-    }
-    return fields;
-}
-
-
-std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
-{
-    if (flush)
-        to.flush();
-
-    while (true) {
-
-        auto msg = readNum<uint64_t>(from);
-
-        if (msg == STDERR_WRITE) {
-            string s = readString(from);
-            if (!sink) throw Error("no sink");
-            (*sink)(s);
-        }
-
-        else if (msg == STDERR_READ) {
-            if (!source) throw Error("no source");
-            size_t len = readNum<size_t>(from);
-            auto buf = std::make_unique<unsigned char[]>(len);
-            writeString(buf.get(), source->read(buf.get(), len), to);
-            to.flush();
-        }
-
-        else if (msg == STDERR_ERROR) {
-            if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
-                return std::make_exception_ptr(readError(from));
-            } else {
-                string error = readString(from);
-                unsigned int status = readInt(from);
-                return std::make_exception_ptr(Error(status, error));
-            }
-        }
-
-        else if (msg == STDERR_NEXT)
-            printError(chomp(readString(from)));
-
-        else if (msg == STDERR_START_ACTIVITY) {
-            auto act = readNum<ActivityId>(from);
-            auto lvl = (Verbosity) readInt(from);
-            auto type = (ActivityType) readInt(from);
-            auto s = readString(from);
-            auto fields = readFields(from);
-            auto parent = readNum<ActivityId>(from);
-            logger->startActivity(act, lvl, type, s, fields, parent);
-        }
-
-        else if (msg == STDERR_STOP_ACTIVITY) {
-            auto act = readNum<ActivityId>(from);
-            logger->stopActivity(act);
-        }
-
-        else if (msg == STDERR_RESULT) {
-            auto act = readNum<ActivityId>(from);
-            auto type = (ResultType) readInt(from);
-            auto fields = readFields(from);
-            logger->result(act, type, fields);
-        }
-
-        else if (msg == STDERR_LAST)
-            break;
-
-        else
-            throw Error("got unknown message type %x from Nix daemon", msg);
-    }
-
-    return nullptr;
-}
-
-void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
-{
-    (*this)->to.flush();
-
-    std::exception_ptr ex;
-
-    /* Handle log messages / exceptions from the remote on a
-        separate thread. */
-    std::thread stderrThread([&]()
-    {
-        try {
-            processStderr(nullptr, nullptr, false);
-        } catch (...) {
-            ex = std::current_exception();
-        }
-    });
-
-    Finally joinStderrThread([&]()
-    {
-        if (stderrThread.joinable()) {
-            stderrThread.join();
-            if (ex) {
-                try {
-                    std::rethrow_exception(ex);
-                } catch (...) {
-                    ignoreException();
-                }
-            }
-        }
-    });
-
-    {
-        FramedSink sink((*this)->to, ex);
-        fun(sink);
-        sink.flush();
-    }
-
-    stderrThread.join();
-    if (ex)
-        std::rethrow_exception(ex);
-
-}
-
 static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
 
 }
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
index 554fb6bed..e5de104c9 100644
--- a/src/libstore/uds-remote-store.hh
+++ b/src/libstore/uds-remote-store.hh
@@ -1,161 +1,10 @@
 #pragma once
 
-#include <limits>
-#include <string>
-
-#include "store-api.hh"
+#include "remote-store.hh"
 #include "local-fs-store.hh"
 
-
 namespace nix {
 
-
-class Pipe;
-class Pid;
-struct FdSink;
-struct FdSource;
-template<typename T> class Pool;
-struct ConnectionHandle;
-
-struct RemoteStoreConfig : virtual StoreConfig
-{
-    using StoreConfig::StoreConfig;
-
-    const Setting<int> maxConnections{(StoreConfig*) this, 1,
-            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
-
-    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
-            "max-connection-age", "number of seconds to reuse a connection"};
-};
-
-/* FIXME: RemoteStore is a misnomer - should be something like
-   DaemonStore. */
-class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
-{
-public:
-
-    virtual bool sameMachine() = 0;
-
-    RemoteStore(const Params & params);
-
-    /* Implementations of abstract store API methods. */
-
-    bool isValidPathUncached(const StorePath & path) override;
-
-    StorePathSet queryValidPaths(const StorePathSet & paths,
-        SubstituteFlag maybeSubstitute = NoSubstitute) override;
-
-    StorePathSet queryAllValidPaths() override;
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
-
-    StorePathSet queryValidDerivers(const StorePath & path) override;
-
-    StorePathSet queryDerivationOutputs(const StorePath & path) override;
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
-
-    StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
-
-    void querySubstitutablePathInfos(const StorePathCAMap & paths,
-        SubstitutablePathInfos & infos) override;
-
-    /* Add a content-addressable store path. `dump` will be drained. */
-    ref<const ValidPathInfo> addCAToStore(
-        Source & dump,
-        const string & name,
-        ContentAddressMethod caMethod,
-        const StorePathSet & references,
-        RepairFlag repair);
-
-    /* Add a content-addressable store path. Does not support references. `dump` will be drained. */
-    StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
-
-    void addToStore(const ValidPathInfo & info, Source & nar,
-        RepairFlag repair, CheckSigsFlag checkSigs) override;
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair) override;
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode) override;
-
-    void ensurePath(const StorePath & path) override;
-
-    void addTempRoot(const StorePath & path) override;
-
-    void addIndirectRoot(const Path & path) override;
-
-    void syncWithGC() override;
-
-    Roots findRoots(bool censor) override;
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override;
-
-    void optimiseStore() override;
-
-    bool verifyStore(bool checkContents, RepairFlag repair) override;
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override;
-
-    void connect() override;
-
-    unsigned int getProtocol() override;
-
-    void flushBadConnections();
-
-    struct Connection
-    {
-        AutoCloseFD fd;
-        FdSink to;
-        FdSource from;
-        unsigned int daemonVersion;
-        std::chrono::time_point<std::chrono::steady_clock> startTime;
-
-        virtual ~Connection();
-
-        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
-    };
-
-    ref<Connection> openConnectionWrapper();
-
-protected:
-
-    virtual ref<Connection> openConnection() = 0;
-
-    void initConnection(Connection & conn);
-
-    ref<Pool<Connection>> connections;
-
-    virtual void setOptions(Connection & conn);
-
-    ConnectionHandle getConnection();
-
-    friend struct ConnectionHandle;
-
-    virtual ref<FSAccessor> getFSAccessor() override;
-
-    virtual void narFromPath(const StorePath & path, Sink & sink) override;
-
-    ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
-
-private:
-
-    std::atomic_bool failed{false};
-
-};
-
 struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
 {
     UDSRemoteStoreConfig(const Store::Params & params)
@@ -200,5 +49,4 @@ private:
     std::optional<std::string> path;
 };
 
-
 }

From eee18f88dd83be2810cd7090738e5949745dbefa Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Mon, 12 Oct 2020 16:06:38 +0000
Subject: [PATCH 376/882] Handle amount of disk space saved by hard linking
 being negative

Fixes bogus messages like "currently hard linking saves 17592186044416.00 MiB".
---
 src/libstore/gc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 7a04b2f89..bc692ca42 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -683,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
     struct stat st;
     if (stat(linksDir.c_str(), &st) == -1)
         throw SysError("statting '%1%'", linksDir);
-    auto overhead = st.st_blocks * 512ULL;
+    int64_t overhead = st.st_blocks * 512ULL;
 
     printInfo("note: currently hard linking saves %.2f MiB",
         ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));

From 5a97621d6deebaaef24c1523c42647bfa43b01f6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:07:51 +0000
Subject: [PATCH 377/882] Prepare for build/*.hh headers

---
 src/libstore/local.mk | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 212a725af..dfe1e2cc4 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -32,7 +32,7 @@ ifeq ($(HAVE_SECCOMP), 1)
 endif
 
 libstore_CXXFLAGS += \
- -I src/libutil -I src/libstore \
+ -I src/libutil -I src/libstore -I src/libstore/build \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \
@@ -64,3 +64,6 @@ $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)
 
 $(foreach i, $(wildcard src/libstore/builtins/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
+
+$(foreach i, $(wildcard src/libstore/build/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/build, 0644)))

From f7099965bfa1ec00cce28f6ed0d5daf3bc093759 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:08:52 +0000
Subject: [PATCH 378/882] Change .cc files to use split build headers

---
 src/libstore/build/derivation-goal.cc   | 17 ++++++++++++++++-
 src/libstore/build/goal.cc              |  3 ++-
 src/libstore/build/hook-instance.cc     |  3 ++-
 src/libstore/build/local-store-build.cc |  5 ++++-
 src/libstore/build/substitution-goal.cc |  3 ++-
 src/libstore/build/worker.cc            |  6 +++++-
 6 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index bb8921759..241e52116 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1,4 +1,6 @@
-#include "build.hh"
+#include "derivation-goal.hh"
+#include "hook-instance.hh"
+#include "worker.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
 #include "references.hh"
@@ -12,6 +14,9 @@
 #include "topo-sort.hh"
 #include "callback.hh"
 
+#include <regex>
+#include <queue>
+
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -140,6 +145,16 @@ DerivationGoal::~DerivationGoal()
 }
 
 
+string DerivationGoal::key()
+{
+    /* Ensure that derivations get built in order of their name,
+       i.e. a derivation named "aardvark" always comes before
+       "baboon". And substitution goals always happen before
+       derivation goals (due to "b$"). */
+    return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
+}
+
+
 inline bool DerivationGoal::needsHashRewrite()
 {
 #if __linux__
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index 404a430f7..2dd7a4d37 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -1,4 +1,5 @@
-#include "build.hh"
+#include "goal.hh"
+#include "worker.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
index a33d5e22b..0f6f580be 100644
--- a/src/libstore/build/hook-instance.cc
+++ b/src/libstore/build/hook-instance.cc
@@ -1,4 +1,5 @@
-#include "build.hh"
+#include "globals.hh"
+#include "hook-instance.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc
index 407979b1e..a05fb5805 100644
--- a/src/libstore/build/local-store-build.cc
+++ b/src/libstore/build/local-store-build.cc
@@ -1,4 +1,7 @@
-#include "build.hh"
+#include "machines.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+#include "derivation-goal.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index dd2ce94a6..d16584f65 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -1,4 +1,5 @@
-#include "build.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
 #include "nar-info.hh"
 #include "finally.hh"
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 2fc9f6982..5c3fe2f57 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -1,4 +1,8 @@
-#include "build.hh"
+#include "machines.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+#include "derivation-goal.hh"
+#include "hook-instance.hh"
 
 #include <poll.h>
 

From 4eb8c698538e0c212214d22a3c516777d52f921b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:15:32 +0000
Subject: [PATCH 379/882] Rename to hand-hold git (derivation-goal.hh)

---
 src/libstore/{build.hh => build/derivation-goal.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.hh => build/derivation-goal.hh} (100%)

diff --git a/src/libstore/build.hh b/src/libstore/build/derivation-goal.hh
similarity index 100%
rename from src/libstore/build.hh
rename to src/libstore/build/derivation-goal.hh

From 2ce726947adf76e558bd0d05f8e309c827a46b2c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:15:32 +0000
Subject: [PATCH 380/882] Trim derivation-goal.hh

---
 src/libstore/build/derivation-goal.hh | 409 +-------------------------
 1 file changed, 2 insertions(+), 407 deletions(-)

diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 8027c61b1..4976207e0 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -1,312 +1,18 @@
 #pragma once
 
-#include "machines.hh"
 #include "parsed-derivations.hh"
 #include "lock.hh"
 #include "local-store.hh"
-
-#include <memory>
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
+#include "goal.hh"
 
 namespace nix {
 
 using std::map;
 
-
-/* Forward definition. */
-class Worker;
 struct HookInstance;
 
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
-class SubstitutionGoal;
-
 /* Unless we are repairing, we don't both to test validity and just assume it,
    so the choices are `Absent` or `Valid`. */
 enum struct PathStatus {
@@ -554,14 +260,7 @@ public:
 
     void timedOut(Error && ex) override;
 
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
+    string key() override;
 
     void work() override;
 
@@ -677,108 +376,4 @@ private:
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p);
-
 }

From 0d0e345cdc1a12cdd8b14a240d0ecd94269b26a5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:00 +0000
Subject: [PATCH 381/882] Rename to hand-hold git (goal.hh)

---
 src/libstore/{build.hh => build/goal.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.hh => build/goal.hh} (100%)

diff --git a/src/libstore/build.hh b/src/libstore/build/goal.hh
similarity index 100%
rename from src/libstore/build.hh
rename to src/libstore/build/goal.hh

From 8067d32f2a69e8822e80fcf5344d8bc981952b04 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:00 +0000
Subject: [PATCH 382/882] Trim goal.hh

---
 src/libstore/build/goal.hh | 685 +------------------------------------
 1 file changed, 4 insertions(+), 681 deletions(-)

diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 8027c61b1..360c160ce 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -1,34 +1,15 @@
 #pragma once
 
-#include "machines.hh"
-#include "parsed-derivations.hh"
-#include "lock.hh"
-#include "local-store.hh"
-
-#include <memory>
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
+#include "types.hh"
+#include "store-api.hh"
 
 namespace nix {
 
-using std::map;
-
-
 /* Forward definition. */
-class Worker;
-struct HookInstance;
-
+struct Goal;
+struct Worker;
 
 /* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
 typedef std::shared_ptr<Goal> GoalPtr;
 typedef std::weak_ptr<Goal> WeakGoalPtr;
 
@@ -43,8 +24,6 @@ typedef list<WeakGoalPtr> WeakGoals;
 /* A map of paths to goals (and the other way around). */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 
-
-
 struct Goal : public std::enable_shared_from_this<Goal>
 {
     typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
@@ -123,662 +102,6 @@ struct Goal : public std::enable_shared_from_this<Goal>
     void amDone(ExitCode result, std::optional<Error> ex = {});
 };
 
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);
 
 }

From 10b749a156d501658f1138c002eeabafeb62dc53 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:13 +0000
Subject: [PATCH 383/882] Rename to hand-hold git (hook-instance.hh)

---
 src/libstore/{build.hh => build/hook-instance.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.hh => build/hook-instance.hh} (100%)

diff --git a/src/libstore/build.hh b/src/libstore/build/hook-instance.hh
similarity index 100%
rename from src/libstore/build.hh
rename to src/libstore/build/hook-instance.hh

From d585b4c54f5a1b4d24168c5c3e7e0f3c4e903b85 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:13 +0000
Subject: [PATCH 384/882] Trim hook-instance.hh

---
 src/libstore/build/hook-instance.hh | 757 +---------------------------
 1 file changed, 2 insertions(+), 755 deletions(-)

diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh
index 8027c61b1..9e8cff128 100644
--- a/src/libstore/build/hook-instance.hh
+++ b/src/libstore/build/hook-instance.hh
@@ -1,760 +1,10 @@
 #pragma once
 
-#include "machines.hh"
-#include "parsed-derivations.hh"
-#include "lock.hh"
-#include "local-store.hh"
-
-#include <memory>
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
+#include "logging.hh"
+#include "serialise.hh"
 
 namespace nix {
 
-using std::map;
-
-
-/* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
 struct HookInstance
 {
     /* Pipes for talking to the build hook. */
@@ -778,7 +28,4 @@ struct HookInstance
     ~HookInstance();
 };
 
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p);
-
 }

From e77a2344d58ef724289c07bd64c1c2e9c69187c2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:25 +0000
Subject: [PATCH 385/882] Rename to hand-hold git (substitution-goal.hh)

---
 src/libstore/{build.hh => build/substitution-goal.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.hh => build/substitution-goal.hh} (100%)

diff --git a/src/libstore/build.hh b/src/libstore/build/substitution-goal.hh
similarity index 100%
rename from src/libstore/build.hh
rename to src/libstore/build/substitution-goal.hh

From 3ffa3546bd41d73cd69424f9c9f862988e2e6afa Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:25 +0000
Subject: [PATCH 386/882] Trim substitution-goal.hh

---
 src/libstore/build/substitution-goal.hh | 699 +-----------------------
 1 file changed, 2 insertions(+), 697 deletions(-)

diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index 8027c61b1..3ae9a9e6b 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -1,681 +1,12 @@
 #pragma once
 
-#include "machines.hh"
-#include "parsed-derivations.hh"
 #include "lock.hh"
-#include "local-store.hh"
-
-#include <memory>
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
+#include "store-api.hh"
+#include "goal.hh"
 
 namespace nix {
 
-using std::map;
-
-
-/* Forward definition. */
 class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
-class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
-
-typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-
-/* A mapping used to remember for each child process to what goal it
-   belongs, and file descriptors for receiving log data and output
-   path creation commands. */
-struct Child
-{
-    WeakGoalPtr goal;
-    Goal * goal2; // ugly hackery
-    set<int> fds;
-    bool respectTimeouts;
-    bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
-    steady_time_point timeStarted;
-};
-
-
-/* The worker class. */
-class Worker
-{
-private:
-
-    /* Note: the worker should only have strong pointers to the
-       top-level goals. */
-
-    /* The top-level goals of the worker. */
-    Goals topGoals;
-
-    /* Goals that are ready to do some work. */
-    WeakGoals awake;
-
-    /* Goals waiting for a build slot. */
-    WeakGoals wantingToBuild;
-
-    /* Child processes currently running. */
-    std::list<Child> children;
-
-    /* Number of build slots occupied.  This includes local builds and
-       substitutions but not remote builds via the build hook. */
-    unsigned int nrLocalBuilds;
-
-    /* Maps used to prevent multiple instantiations of a goal for the
-       same derivation / path. */
-    WeakGoalMap derivationGoals;
-    WeakGoalMap substitutionGoals;
-
-    /* Goals waiting for busy paths to be unlocked. */
-    WeakGoals waitingForAnyGoal;
-
-    /* Goals sleeping for a few seconds (polling a lock). */
-    WeakGoals waitingForAWhile;
-
-    /* Last time the goals in `waitingForAWhile' where woken up. */
-    steady_time_point lastWokenUp;
-
-    /* Cache for pathContentsGood(). */
-    std::map<StorePath, bool> pathContentsGoodCache;
-
-public:
-
-    const Activity act;
-    const Activity actDerivations;
-    const Activity actSubstitutions;
-
-    /* Set if at least one derivation had a BuildError (i.e. permanent
-       failure). */
-    bool permanentFailure;
-
-    /* Set if at least one derivation had a timeout. */
-    bool timedOut;
-
-    /* Set if at least one derivation fails with a hash mismatch. */
-    bool hashMismatch;
-
-    /* Set if at least one derivation is not deterministic in check mode. */
-    bool checkMismatch;
-
-    LocalStore & store;
-
-    std::unique_ptr<HookInstance> hook;
-
-    uint64_t expectedBuilds = 0;
-    uint64_t doneBuilds = 0;
-    uint64_t failedBuilds = 0;
-    uint64_t runningBuilds = 0;
-
-    uint64_t expectedSubstitutions = 0;
-    uint64_t doneSubstitutions = 0;
-    uint64_t failedSubstitutions = 0;
-    uint64_t runningSubstitutions = 0;
-    uint64_t expectedDownloadSize = 0;
-    uint64_t doneDownloadSize = 0;
-    uint64_t expectedNarSize = 0;
-    uint64_t doneNarSize = 0;
-
-    /* Whether to ask the build hook if it can build a derivation. If
-       it answers with "decline-permanently", we don't try again. */
-    bool tryBuildHook = true;
-
-    Worker(LocalStore & store);
-    ~Worker();
-
-    /* Make a goal (with caching). */
-
-    /* derivation goal */
-private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
-        const StorePath & drvPath, const StringSet & wantedOutputs,
-        std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-public:
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
-        const StorePath & drvPath,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
-        const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-
-    /* substitution goal */
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-
-    /* Remove a dead goal. */
-    void removeGoal(GoalPtr goal);
-
-    /* Wake up a goal (i.e., there is something for it to do). */
-    void wakeUp(GoalPtr goal);
-
-    /* Return the number of local build and substitution processes
-       currently running (but not remote builds via the build
-       hook). */
-    unsigned int getNrLocalBuilds();
-
-    /* Registers a running child process.  `inBuildSlot' means that
-       the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
-        bool inBuildSlot, bool respectTimeouts);
-
-    /* Unregisters a running child process.  `wakeSleepers' should be
-       false if there is no sense in waking up goals that are sleeping
-       because they can't run yet (e.g., there is no free build slot,
-       or the hook would still say `postpone'). */
-    void childTerminated(Goal * goal, bool wakeSleepers = true);
-
-    /* Put `goal' to sleep until a build slot becomes available (which
-       might be right away). */
-    void waitForBuildSlot(GoalPtr goal);
-
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
-       wait for some resource that some other goal is holding. */
-    void waitForAnyGoal(GoalPtr goal);
-
-    /* Wait for a few seconds and then retry this goal.  Used when
-       waiting for a lock held by another process.  This kind of
-       polling is inefficient, but POSIX doesn't really provide a way
-       to wait for multiple locks in the main select() loop. */
-    void waitForAWhile(GoalPtr goal);
-
-    /* Loop until the specified top-level goals have finished. */
-    void run(const Goals & topGoals);
-
-    /* Wait for input to become available. */
-    void waitForInput();
-
-    unsigned int exitStatus();
-
-    /* Check whether the given valid path exists and has the right
-       contents. */
-    bool pathContentsGood(const StorePath & path);
-
-    void markContentsGood(const StorePath & path);
-
-    void updateProgress()
-    {
-        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
-        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
-    }
-};
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
 
 class SubstitutionGoal : public Goal
 {
@@ -755,30 +86,4 @@ public:
     StorePath getStorePath() { return storePath; }
 };
 
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p);
-
 }

From 0e2306204aa2ced6583385016dcceda112becb74 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:48 +0000
Subject: [PATCH 387/882] Rename to hand-hold git (worker.hh)

---
 src/libstore/{build.hh => build/worker.hh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libstore/{build.hh => build/worker.hh} (100%)

diff --git a/src/libstore/build.hh b/src/libstore/build/worker.hh
similarity index 100%
rename from src/libstore/build.hh
rename to src/libstore/build/worker.hh

From 542972f02960b3ea7f4981e9ec8d89d3c0de957d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 12 Oct 2020 17:16:48 +0000
Subject: [PATCH 388/882] Trim worker.hh

---
 src/libstore/build/worker.hh | 597 +----------------------------------
 1 file changed, 4 insertions(+), 593 deletions(-)

diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 8027c61b1..a54316343 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -1,127 +1,14 @@
 #pragma once
 
-#include "machines.hh"
-#include "parsed-derivations.hh"
+#include "types.hh"
 #include "lock.hh"
 #include "local-store.hh"
-
-#include <memory>
-#include <algorithm>
-#include <iostream>
-#include <map>
-#include <sstream>
-#include <thread>
-#include <future>
-#include <chrono>
-#include <regex>
-#include <queue>
+#include "goal.hh"
 
 namespace nix {
 
-using std::map;
-
-
 /* Forward definition. */
-class Worker;
-struct HookInstance;
-
-
-/* A pointer to a goal. */
-struct Goal;
 class DerivationGoal;
-typedef std::shared_ptr<Goal> GoalPtr;
-typedef std::weak_ptr<Goal> WeakGoalPtr;
-
-struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
-};
-
-/* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
-
-/* A map of paths to goals (and the other way around). */
-typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
-
-
-
-struct Goal : public std::enable_shared_from_this<Goal>
-{
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
-
-    /* Backlink to the worker. */
-    Worker & worker;
-
-    /* Goals that this goal is waiting for. */
-    Goals waitees;
-
-    /* Goals waiting for this one to finish.  Must use weak pointers
-       here to prevent cycles. */
-    WeakGoals waiters;
-
-    /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
-
-    /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
-
-    /* Name of this goal for debugging purposes. */
-    string name;
-
-    /* Whether the goal is finished. */
-    ExitCode exitCode;
-
-    /* Exception containing an error message, if any. */
-    std::optional<Error> ex;
-
-    Goal(Worker & worker) : worker(worker)
-    {
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
-
-    virtual ~Goal()
-    {
-        trace("goal destroyed");
-    }
-
-    virtual void work() = 0;
-
-    void addWaitee(GoalPtr waitee);
-
-    virtual void waiteeDone(GoalPtr waitee, ExitCode result);
-
-    virtual void handleChildOutput(int fd, const string & data)
-    {
-        abort();
-    }
-
-    virtual void handleEOF(int fd)
-    {
-        abort();
-    }
-
-    void trace(const FormatOrString & fs);
-
-    string getName()
-    {
-        return name;
-    }
-
-    /* Callback in case of a timeout.  It should wake up its waiters,
-       get rid of any running child processes that are being monitored
-       by the worker (important!), etc. */
-    virtual void timedOut(Error && ex) = 0;
-
-    virtual string key() = 0;
-
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-};
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
@@ -140,6 +27,8 @@ struct Child
     steady_time_point timeStarted;
 };
 
+/* Forward definition. */
+struct HookInstance;
 
 /* The worker class. */
 class Worker
@@ -303,482 +192,4 @@ public:
     }
 };
 
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-class SubstitutionGoal;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    std::optional<InitialOutputStatus> known;
-};
-
-class DerivationGoal : public Goal
-{
-private:
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
-public:
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
-
-    void timedOut(Error && ex) override;
-
-    string key() override
-    {
-        /* Ensure that derivations get built in order of their name,
-           i.e. a derivation named "aardvark" always comes before
-           "baboon". And substitution goals always happen before
-           derivation goals (due to "b$"). */
-        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-    }
-
-    void work() override;
-
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-private:
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    void tryLocalBuild();
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
-
-    friend int childEntry(void *);
-
-    /* Check that the derivation outputs all exist and register them
-       as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-
-    /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
-
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-};
-
-class SubstitutionGoal : public Goal
-{
-    friend class Worker;
-
-private:
-    /* The store path that should be realised through a substitute. */
-    StorePath storePath;
-
-    /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
-    std::optional<StorePath> subPath;
-
-    /* The remaining substituters. */
-    std::list<ref<Store>> subs;
-
-    /* The current substituter. */
-    std::shared_ptr<Store> sub;
-
-    /* Whether a substituter failed. */
-    bool substituterFailed = false;
-
-    /* Path info returned by the substituter's query info operation. */
-    std::shared_ptr<const ValidPathInfo> info;
-
-    /* Pipe for the substituter's standard output. */
-    Pipe outPipe;
-
-    /* The substituter thread. */
-    std::thread thr;
-
-    std::promise<void> promise;
-
-    /* Whether to try to repair a valid path. */
-    RepairFlag repair;
-
-    /* Location where we're downloading the substitute.  Differs from
-       storePath when doing a repair. */
-    Path destPath;
-
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
-
-    typedef void (SubstitutionGoal::*GoalState)();
-    GoalState state;
-
-    /* Content address for recomputing store path */
-    std::optional<ContentAddress> ca;
-
-public:
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
-
-    void timedOut(Error && ex) override { abort(); };
-
-    string key() override
-    {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
-        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
-    }
-
-    void work() override;
-
-    /* The states. */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
-
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
-};
-
-struct HookInstance
-{
-    /* Pipes for talking to the build hook. */
-    Pipe toHook;
-
-    /* Pipe for the hook's standard output/error. */
-    Pipe fromHook;
-
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* The process ID of the hook. */
-    Pid pid;
-
-    FdSink sink;
-
-    std::map<ActivityId, Activity> activities;
-
-    HookInstance();
-
-    ~HookInstance();
-};
-
-
-void addToWeakGoals(WeakGoals & goals, GoalPtr p);
-
 }

From 55592b253f3dddb121c1072ca584e95c37729b6d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 13 Oct 2020 18:04:24 +0000
Subject: [PATCH 389/882] Add some more docs

---
 src/libstore/build/worker.hh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 07c0c0f16..f8bacc514 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -11,6 +11,11 @@ namespace nix {
 class DerivationGoal;
 class SubstitutionGoal;
 
+/* Workaround for not being able to declare a something like
+
+     class SubstitutionGoal : public Goal;
+
+   even when Goal is a complete type; */
 GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;

From 11882d7c7ce3b6dc51dd7c0536f9662dc254ac0a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 14 Oct 2020 12:20:58 +0200
Subject: [PATCH 390/882] Create /etc/passwd *after* figuring out the sandbox
 uid/gid

Fixes build failures like

  # nix log /nix/store/gjaa0psfcmqvw7ivggsncx9w364p3s8s-sshd.conf-validated.drv
  No user exists for uid 30012
---
 src/libstore/build/derivation-goal.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 241e52116..fda05f0e9 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1420,12 +1420,6 @@ void DerivationGoal::startBuilder()
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
 
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
         /* Declare the build user's group so that programs get a consistent
            view of the system (e.g., "id -gn"). */
         writeFile(chrootRootDir + "/etc/group",
@@ -1730,6 +1724,14 @@ void DerivationGoal::startBuilder()
                 throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
         }
 
+        /* Now that we now the sandbox uid, we can write
+           /etc/passwd. */
+        writeFile(chrootRootDir + "/etc/passwd", fmt(
+                "root:x:0:0:Nix build user:%3%:/noshell\n"
+                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
+                "nobody:x:65534:65534:Nobody:/:/noshell\n",
+                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
+
         /* Save the mount namespace of the child. We have to do this
            *before* the child does a chroot. */
         sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);

From f6ed1a96b397f0345af029127cfde86bcd0247d2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 Oct 2020 18:54:36 +0000
Subject: [PATCH 391/882] `build-static` -> `buildStatic` in Nix's flake

---
 flake.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index a5710f608..0602861fa 100644
--- a/flake.nix
+++ b/flake.nix
@@ -228,7 +228,7 @@
         # Binary package for various platforms.
         build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix);
 
-        build-static = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static);
+        buildStatic = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static);
 
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
@@ -441,7 +441,7 @@
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
       } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) {
-        build-static = self.hydraJobs.build-static.${system};
+        buildStatic = self.hydraJobs.buildStatic.${system};
       });
 
       packages = forAllSystems (system: {

From 64be1c15c229facfa849f5667f603cce951a8488 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 Oct 2020 19:05:06 +0000
Subject: [PATCH 392/882] Add missing include for MAX_PATH

And remove one that we didn't actually need to add
---
 src/libstore/globals.cc    | 1 -
 src/libutil/tests/tests.cc | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index f4a4f348f..1238dc530 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -9,7 +9,6 @@
 #include <thread>
 #include <dlfcn.h>
 #include <sys/utsname.h>
-#include <unordered_set>
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 8e77ccbe1..ffba832d8 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -1,6 +1,7 @@
 #include "util.hh"
 #include "types.hh"
 
+#include <limits.h>
 #include <gtest/gtest.h>
 
 namespace nix {

From 48ce62737750215208804947a1509ad7d26f6214 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 Oct 2020 20:13:01 +0000
Subject: [PATCH 393/882] Make a better -lz hack

Per the comments, the underlying issue is
https://github.com/libarchive/libarchive/issues/1446, knowing this
allows the hack to be much more targetted.
---
 configure.ac   | 4 ++++
 mk/programs.mk | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index eecb107d7..39306b953 100644
--- a/configure.ac
+++ b/configure.ac
@@ -179,6 +179,10 @@ AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 # Checks for libarchive
 PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
+# Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
+if test "$shared" != yes; then
+    LIBARCHIVE_LIBS+=' -lz'
+fi
 
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
diff --git a/mk/programs.mk b/mk/programs.mk
index a96ff56af..3fa9685c3 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -32,7 +32,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) -lz
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
 

From 257090d030508160bb380554433f0c3622470c17 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 Oct 2020 21:49:49 +0000
Subject: [PATCH 394/882] Bump Nixpkgs to hopefully fix linkrot

---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 822a73332..ddff58979 100644
--- a/flake.lock
+++ b/flake.lock
@@ -18,11 +18,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1591633336,
-        "narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=",
+        "lastModified": 1602604700,
+        "narHash": "sha256-TSfAZX0czPf1P8xnnGFXcoeoM9I5CaFjAdNP63W9DCY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "70717a337f7ae4e486ba71a500367cad697e5f09",
+        "rev": "3a10a004bb5802d5f23c58886722e4239705e733",
         "type": "github"
       },
       "original": {

From 5cfdf16dd62dcfa2eb32b4e9ae390cb99d683907 Mon Sep 17 00:00:00 2001
From: aszlig <aszlig@nix.build>
Date: Sat, 17 Oct 2020 22:08:18 +0200
Subject: [PATCH 395/882] Convert VM tests to Python

Perl-based tests are deprecated since NixOS 20.03 and subsequently got
removed in NixOS 20.09, which effectively means that tests are going to
fail as soon as we build it with NixOS 20.09 or anything newer.

I've put "# fmt: off" at the start of every testScript, because
formatting with Black really messes up indentation and I don't think it
really adds anything in value or readability for inlined Python scripts.

Signed-off-by: aszlig <aszlig@nix.build>
---
 tests/github-flakes.nix    |  45 +++++------
 tests/nix-copy-closure.nix |  69 +++++++++--------
 tests/remote-builds.nix    |  70 ++++++++---------
 tests/setuid.nix           | 149 ++++++++++++++++++++-----------------
 4 files changed, 178 insertions(+), 155 deletions(-)

diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
index a47610d9a..2de3e2bc0 100644
--- a/tests/github-flakes.nix
+++ b/tests/github-flakes.nix
@@ -1,6 +1,6 @@
 { nixpkgs, system, overlay }:
 
-with import (nixpkgs + "/nixos/lib/testing.nix") {
+with import (nixpkgs + "/nixos/lib/testing-python.nix") {
   inherit system;
   extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
 };
@@ -113,36 +113,37 @@ makeTest (
         };
     };
 
-  testScript = { nodes }:
-    ''
-      use POSIX qw(strftime);
+  testScript = { nodes }: ''
+    # fmt: off
+    import json
+    import time
 
-      startAll;
+    start_all()
 
-      $github->waitForUnit("httpd.service");
+    github.wait_for_unit("httpd.service")
 
-      $client->succeed("curl -v https://github.com/ >&2");
+    client.succeed("curl -v https://github.com/ >&2")
+    client.succeed("nix registry list | grep nixpkgs")
 
-      $client->succeed("nix registry list | grep nixpkgs");
+    rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")
+    assert rev.strip() == "${nixpkgs.rev}", "revision mismatch"
 
-      $client->succeed("nix flake info nixpkgs --json | jq -r .revision") eq "${nixpkgs.rev}\n"
-        or die "revision mismatch";
+    client.succeed("nix registry pin nixpkgs")
 
-      $client->succeed("nix registry pin nixpkgs");
+    client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2")
 
-      $client->succeed("nix flake info nixpkgs --tarball-ttl 0 >&2");
+    # Shut down the web server. The flake should be cached on the client.
+    github.succeed("systemctl stop httpd.service")
 
-      # Shut down the web server. The flake should be cached on the client.
-      $github->succeed("systemctl stop httpd.service");
+    info = json.loads(client.succeed("nix flake info nixpkgs --json"))
+    date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified']))
+    assert date == "${nixpkgs.lastModifiedDate}", "time mismatch"
 
-      my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified");
-      strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModifiedDate}" or die "time mismatch";
+    client.succeed("nix build nixpkgs#hello")
 
-      $client->succeed("nix build nixpkgs#hello");
-
-      # The build shouldn't fail even with --tarball-ttl 0 (the server
-      # being down should not be a fatal error).
-      $client->succeed("nix build nixpkgs#fuse --tarball-ttl 0");
-    '';
+    # The build shouldn't fail even with --tarball-ttl 0 (the server
+    # being down should not be a fatal error).
+    client.succeed("nix build nixpkgs#fuse --tarball-ttl 0")
+  '';
 
 })
diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index 9c9d119b7..68f9c70b3 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -2,7 +2,7 @@
 
 { nixpkgs, system, overlay }:
 
-with import (nixpkgs + "/nixos/lib/testing.nix") {
+with import (nixpkgs + "/nixos/lib/testing-python.nix") {
   inherit system;
   extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
 };
@@ -25,41 +25,46 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
         };
     };
 
-  testScript = { nodes }:
-    ''
-      startAll;
+  testScript = { nodes }: ''
+    # fmt: off
+    import subprocess
 
-      # Create an SSH key on the client.
-      my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
-      $client->succeed("mkdir -m 700 /root/.ssh");
-      $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
-      $client->succeed("chmod 600 /root/.ssh/id_ed25519");
+    start_all()
 
-      # Install the SSH key on the server.
-      $server->succeed("mkdir -m 700 /root/.ssh");
-      $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys");
-      $server->waitForUnit("sshd");
-      $client->waitForUnit("network.target");
-      $client->succeed("ssh -o StrictHostKeyChecking=no " . $server->name() . " 'echo hello world'");
+    # Create an SSH key on the client.
+    subprocess.run([
+      "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", ""
+    ], capture_output=True, check=True)
 
-      # Copy the closure of package A from the client to the server.
-      $server->fail("nix-store --check-validity ${pkgA}");
-      $client->succeed("nix-copy-closure --to server --gzip ${pkgA} >&2");
-      $server->succeed("nix-store --check-validity ${pkgA}");
+    client.succeed("mkdir -m 700 /root/.ssh")
+    client.copy_from_host("key", "/root/.ssh/id_ed25519")
+    client.succeed("chmod 600 /root/.ssh/id_ed25519")
 
-      # Copy the closure of package B from the server to the client.
-      $client->fail("nix-store --check-validity ${pkgB}");
-      $client->succeed("nix-copy-closure --from server --gzip ${pkgB} >&2");
-      $client->succeed("nix-store --check-validity ${pkgB}");
+    # Install the SSH key on the server.
+    server.succeed("mkdir -m 700 /root/.ssh")
+    server.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
+    server.wait_for_unit("sshd")
+    client.wait_for_unit("network.target")
+    client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'")
 
-      # Copy the closure of package C via the SSH substituter.
-      $client->fail("nix-store -r ${pkgC}");
-      # FIXME
-      #$client->succeed(
-      #  "nix-store --option use-ssh-substituter true"
-      #  . " --option ssh-substituter-hosts root\@server"
-      #  . " -r ${pkgC} >&2");
-      #$client->succeed("nix-store --check-validity ${pkgC}");
-    '';
+    # Copy the closure of package A from the client to the server.
+    server.fail("nix-store --check-validity ${pkgA}")
+    client.succeed("nix-copy-closure --to server --gzip ${pkgA} >&2")
+    server.succeed("nix-store --check-validity ${pkgA}")
 
+    # Copy the closure of package B from the server to the client.
+    client.fail("nix-store --check-validity ${pkgB}")
+    client.succeed("nix-copy-closure --from server --gzip ${pkgB} >&2")
+    client.succeed("nix-store --check-validity ${pkgB}")
+
+    # Copy the closure of package C via the SSH substituter.
+    client.fail("nix-store -r ${pkgC}")
+    # FIXME
+    # client.succeed(
+    #   "nix-store --option use-ssh-substituter true"
+    #   " --option ssh-substituter-hosts root\@server"
+    #   " -r ${pkgC} >&2"
+    # )
+    # client.succeed("nix-store --check-validity ${pkgC}")
+  '';
 })
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 153956619..305c82394 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -2,7 +2,7 @@
 
 { nixpkgs, system, overlay }:
 
-with import (nixpkgs + "/nixos/lib/testing.nix") {
+with import (nixpkgs + "/nixos/lib/testing-python.nix") {
   inherit system;
   extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
 };
@@ -66,44 +66,46 @@ in
         };
     };
 
-  testScript = { nodes }:
-    ''
-      startAll;
+  testScript = { nodes }: ''
+    # fmt: off
+    import subprocess
 
-      # Create an SSH key on the client.
-      my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
-      $client->succeed("mkdir -p -m 700 /root/.ssh");
-      $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
-      $client->succeed("chmod 600 /root/.ssh/id_ed25519");
+    start_all()
 
-      # Install the SSH key on the builders.
-      $client->waitForUnit("network.target");
-      foreach my $builder ($builder1, $builder2) {
-          $builder->succeed("mkdir -p -m 700 /root/.ssh");
-          $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys");
-          $builder->waitForUnit("sshd");
-          $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'");
-      }
+    # Create an SSH key on the client.
+    subprocess.run([
+      "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", ""
+    ], capture_output=True, check=True)
+    client.succeed("mkdir -p -m 700 /root/.ssh")
+    client.copy_from_host("key", "/root/.ssh/id_ed25519")
+    client.succeed("chmod 600 /root/.ssh/id_ed25519")
 
-      # Perform a build and check that it was performed on the builder.
-      my $out = $client->succeed(
-        "nix-build ${expr nodes.client.config 1} 2> build-output",
-        "grep -q Hello build-output"
-      );
-      $builder1->succeed("test -e $out");
+    # Install the SSH key on the builders.
+    client.wait_for_unit("network.target")
+    for builder in [builder1, builder2]:
+      builder.succeed("mkdir -p -m 700 /root/.ssh")
+      builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
+      builder.wait_for_unit("sshd")
+      client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'")
 
-      # And a parallel build.
-      my ($out1, $out2) = split /\s/,
-          $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out');
-      $builder1->succeed("test -e $out1 -o -e $out2");
-      $builder2->succeed("test -e $out1 -o -e $out2");
+    # Perform a build and check that it was performed on the builder.
+    out = client.succeed(
+      "nix-build ${expr nodes.client.config 1} 2> build-output",
+      "grep -q Hello build-output"
+    )
+    builder1.succeed(f"test -e {out}")
 
-      # And a failing build.
-      $client->fail("nix-build ${expr nodes.client.config 5}");
+    # And a parallel build.
+    paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out')
+    out1, out2 = paths.split()
+    builder1.succeed(f"test -e {out1} -o -e {out2}")
+    builder2.succeed(f"test -e {out1} -o -e {out2}")
 
-      # Test whether the build hook automatically skips unavailable builders.
-      $builder1->block;
-      $client->succeed("nix-build ${expr nodes.client.config 4}");
-    '';
+    # And a failing build.
+    client.fail("nix-build ${expr nodes.client.config 5}")
 
+    # Test whether the build hook automatically skips unavailable builders.
+    builder1.block()
+    client.succeed("nix-build ${expr nodes.client.config 4}")
+  '';
 })
diff --git a/tests/setuid.nix b/tests/setuid.nix
index 6f2f7d392..8d31a18be 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -2,7 +2,7 @@
 
 { nixpkgs, system, overlay }:
 
-with import (nixpkgs + "/nixos/lib/testing.nix") {
+with import (nixpkgs + "/nixos/lib/testing-python.nix") {
   inherit system;
   extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
 };
@@ -17,94 +17,109 @@ makeTest {
       virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ];
     };
 
-  testScript = { nodes }:
-    ''
-      startAll;
+  testScript = { nodes }: ''
+    # fmt: off
+    start_all()
 
-      # Copying to /tmp should succeed.
-      $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-      ")\' ');
+    # Copying to /tmp should succeed.
+    machine.succeed(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" {} "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      # Creating a setuid binary should fail.
-      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        chmod 4755 /tmp/id
-      ")\' ');
+    # Creating a setuid binary should fail.
+    machine.fail(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" {} "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      chmod 4755 /tmp/id
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      # Creating a setgid binary should fail.
-      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        chmod 2755 /tmp/id
-      ")\' ');
+    # Creating a setgid binary should fail.
+    machine.fail(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" {} "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      chmod 2755 /tmp/id
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      # The checks should also work on 32-bit binaries.
-      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        chmod 2755 /tmp/id
-      ")\' ');
+    # The checks should also work on 32-bit binaries.
+    machine.fail(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      chmod 2755 /tmp/id
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      # The tests above use fchmodat(). Test chmod() as well.
-      $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        perl -e \"chmod 0666, qw(/tmp/id) or die\"
-      ")\' ');
+    # The tests above use fchmodat(). Test chmod() as well.
+    machine.succeed(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      perl -e \"chmod 0666, qw(/tmp/id) or die\"
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 666 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        perl -e \"chmod 04755, qw(/tmp/id) or die\"
-      ")\' ');
+    machine.fail(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      perl -e \"chmod 04755, qw(/tmp/id) or die\"
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      # And test fchmod().
-      $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\"
-      ")\' ');
+    # And test fchmod().
+    machine.succeed(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\"
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]');
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]')
 
-      $machine->succeed("rm /tmp/id");
+    machine.succeed("rm /tmp/id")
 
-      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
-        mkdir -p $out
-        cp ${pkgs.coreutils}/bin/id /tmp/id
-        perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\"
-      ")\' ');
+    machine.fail(r"""
+    nix-build --no-sandbox -E '(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      mkdir -p $out
+      cp ${pkgs.coreutils}/bin/id /tmp/id
+      perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\"
+    ")'
+    """.strip())
 
-      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
-
-      $machine->succeed("rm /tmp/id");
-    '';
+    machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]')
 
+    machine.succeed("rm /tmp/id")
+  '';
 }

From cfa26cf18135ec1a16ce72319b0bc8b297600ef1 Mon Sep 17 00:00:00 2001
From: aszlig <aszlig@nix.build>
Date: Sat, 17 Oct 2020 23:34:38 +0200
Subject: [PATCH 396/882] tests: Add names to VM tests

Having vm-test-run-unnamed for all the test derivation doesn't look very
nice, so in order to better distinguish them from their store path,
let's actually give them proper names.

Signed-off-by: aszlig <aszlig@nix.build>
---
 tests/github-flakes.nix    | 1 +
 tests/nix-copy-closure.nix | 1 +
 tests/remote-builds.nix    | 1 +
 tests/setuid.nix           | 1 +
 4 files changed, 4 insertions(+)

diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
index 2de3e2bc0..7ac397d81 100644
--- a/tests/github-flakes.nix
+++ b/tests/github-flakes.nix
@@ -64,6 +64,7 @@ in
 makeTest (
 
 {
+  name = "github-flakes";
 
   nodes =
     { # Impersonate github.com and api.github.com.
diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index 68f9c70b3..e5f6a0f12 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -8,6 +8,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") {
 };
 
 makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
+  name = "nix-copy-closure";
 
   nodes =
     { client =
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 305c82394..b9e7352c0 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -36,6 +36,7 @@ let
 in
 
 {
+  name = "remote-builds";
 
   nodes =
     { builder1 = builder;
diff --git a/tests/setuid.nix b/tests/setuid.nix
index 8d31a18be..35eb304ed 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -8,6 +8,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") {
 };
 
 makeTest {
+  name = "setuid";
 
   machine =
     { config, lib, pkgs, ... }:

From e6f8ae56d82813edbcf09fda58305de47369f964 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 17 Oct 2020 21:45:31 +0000
Subject: [PATCH 397/882] tab -> space

---
 src/libstore/build/worker.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 97ffb9a34..4d3df26f3 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -455,7 +455,7 @@ void Worker::markContentsGood(const StorePath & path)
 
 
 GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal) {
-	return subGoal;
+    return subGoal;
 }
 
 }

From 57d0432b395cc4d70792d2df5794ff2e0dd02d3d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 17 Oct 2020 21:47:52 +0000
Subject: [PATCH 398/882] Just use `auto` in two places.

---
 src/libstore/build/worker.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 4d3df26f3..17c10cd71 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -90,10 +90,10 @@ template<typename G>
 static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
-    for (typename std::map<StorePath, std::weak_ptr<G>>::iterator i = goalMap.begin();
+    for (auto i = goalMap.begin();
          i != goalMap.end(); )
         if (i->second.lock() == goal) {
-            typename std::map<StorePath, std::weak_ptr<G>>::iterator j = i; ++j;
+            auto j = i; ++j;
             goalMap.erase(i);
             i = j;
         }

From 7ed46c15744461534478e2ed0aa25a2b2e536c6f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 17 Oct 2020 21:50:12 +0000
Subject: [PATCH 399/882] Explain that `upcast_goal` is still a static cast

---
 src/libstore/build/worker.hh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index f8bacc514..3a53a8def 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -15,7 +15,11 @@ class SubstitutionGoal;
 
      class SubstitutionGoal : public Goal;
 
-   even when Goal is a complete type; */
+   even when Goal is a complete type.
+
+   This is still a static cast. The purpose of exporting it is to define it in
+   a place where `SubstitutionGoal` is concrete, and use it in a place where it
+   is opaque. */
 GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;

From a53438d18f1faddb9f15c6980c828eb11820284c Mon Sep 17 00:00:00 2001
From: Stefan Frijters <sfrijters@gmail.com>
Date: Sun, 18 Oct 2020 14:14:37 +0200
Subject: [PATCH 400/882] doc: nix-shell in pure mode does *not* source user
 bashrc

---
 doc/manual/src/command-ref/nix-shell.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 45a5ff08c..d1266930e 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -76,8 +76,8 @@ All options not listed here are passed to `nix-store
     cleared before the interactive shell is started, so you get an
     environment that more closely corresponds to the “real” Nix build. A
     few variables, in particular `HOME`, `USER` and `DISPLAY`, are
-    retained. Note that `~/.bashrc` and (depending on your Bash
-    installation) `/etc/bashrc` are still sourced, so any variables set
+    retained. Note that (depending on your Bash
+    installation) `/etc/bashrc` is still sourced, so any variables set
     there will affect the interactive shell.
 
   - `--packages` / `-p` *packages*…  

From 94f1e4a441ee64bcc7a961d941ec901de750d880 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 18 Oct 2020 14:26:23 +0200
Subject: [PATCH 401/882] Typo

---
 src/libstore/build/goal.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 360c160ce..0781a9d38 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -46,7 +46,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
     unsigned int nrNoSubstituters;
 
     /* Number of substitution goals we are/were waiting for that
-       failed because othey had unsubstitutable references. */
+       failed because they had unsubstitutable references. */
     unsigned int nrIncompleteClosure;
 
     /* Name of this goal for debugging purposes. */

From ea8d32020e1e24576cf5e8ff1a835f149dbd81c3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 18 Oct 2020 00:43:52 +0200
Subject: [PATCH 402/882] Tests for #3964

---
 tests/binary-cache.sh | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index fe4ddec8d..e14cf882e 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -239,3 +239,37 @@ nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
 diff -u \
     <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \
     <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S)
+
+# Test against issue https://github.com/NixOS/nix/issues/3964
+#
+expr='
+  with import ./config.nix;
+  mkDerivation {
+    name = "multi-output";
+    buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref";
+    outputs = ["out" "doc"];
+  }
+'
+outPath=$(nix-build --no-out-link -E "$expr")
+docPath=$(nix-store -q --references $outPath)
+
+# $ nix-store -q --tree $outPath
+# ...-multi-output
+# +---...-multi-output-doc
+
+nix copy --to "file://$cacheDir" $outPath
+( echo $outPath $docPath
+  find $cacheDir
+) >/tmp/blurb
+
+hashpart() {
+  basename "$1" | cut -c1-32
+}
+
+# break the closure of out by removing doc
+rm $cacheDir/$(hashpart $docPath).narinfo
+
+nix-store --delete $outPath $docPath
+# -vvv is the level that logs during the loop
+timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \
+  --option trusted-binary-caches "file://$cacheDir"  --no-require-sigs

From bd9eb5c743faf1b3c33f4e1c2ccf317977d4be9d Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 18 Oct 2020 14:21:53 +0200
Subject: [PATCH 403/882] DerivationGoal: only retry if output closure
 incomplete is only problem

---
 src/libstore/build/derivation-goal.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index fda05f0e9..1c9217537 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -330,8 +330,13 @@ void DerivationGoal::outputsSubstitutionTried()
 
     /*  If the substitutes form an incomplete closure, then we should
         build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0) retrySubstitution = true;
+        can still use the substitutes for this derivation itself.
+
+        If the nrIncompleteClosure != nrFailed, we have another issue as well.
+        In particular, it may be the case that the hole in the closure is
+        an output of the current derivation, which causes a loop if retried.
+     */
+    if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true;
 
     nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
 

From 93bd014c8c416580d8eeaa487e180d6c2549028c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Oct 2020 20:32:46 +0200
Subject: [PATCH 404/882] Update .gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index b087cd8d5..c51582cf0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,7 +24,6 @@ perl/Makefile.config
 
 # /scripts/
 /scripts/nix-profile.sh
-/scripts/nix-copy-closure
 /scripts/nix-reduce-build
 /scripts/nix-http-export.cgi
 /scripts/nix-profile-daemon.sh

From 20a7d8d23a19530207378fc74ddd558232240f43 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Oct 2020 20:32:59 +0200
Subject: [PATCH 405/882] Add some missing clean-files

---
 misc/systemd/local.mk | 2 ++
 misc/upstart/local.mk | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk
index 004549fd2..785db52a4 100644
--- a/misc/systemd/local.mk
+++ b/misc/systemd/local.mk
@@ -2,4 +2,6 @@ ifeq ($(OS), Linux)
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
 
+  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service
+
 endif
diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk
index a73dc061e..5071676dc 100644
--- a/misc/upstart/local.mk
+++ b/misc/upstart/local.mk
@@ -2,4 +2,6 @@ ifeq ($(OS), Linux)
 
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644)))
 
+  clean-files += $(d)/nix-daemon.conf
+
 endif

From 532d2bc1890d59a31e212eaea8d351a8cca66e19 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Oct 2020 21:20:35 +0200
Subject: [PATCH 406/882] flake.lock: Update

Flake input changes:

* Updated 'nixpkgs': 'github:NixOS/nixpkgs/3a10a004bb5802d5f23c58886722e4239705e733' -> 'github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31'
---
 flake.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index ddff58979..9f8c788ac 100644
--- a/flake.lock
+++ b/flake.lock
@@ -18,16 +18,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1602604700,
-        "narHash": "sha256-TSfAZX0czPf1P8xnnGFXcoeoM9I5CaFjAdNP63W9DCY=",
+        "lastModified": 1602702596,
+        "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3a10a004bb5802d5f23c58886722e4239705e733",
+        "rev": "ad0d20345219790533ebe06571f82ed6b034db31",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "nixos-20.03-small",
+        "ref": "nixos-20.09-small",
         "type": "indirect"
       }
     },

From 62cf1d815a6c55853e9a4041e85b7e6e0b04d7e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Oct 2020 21:31:27 +0200
Subject: [PATCH 407/882] Switch to Nixpkgs 20.09

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 0602861fa..22ea2911e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,7 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
+  inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
   outputs = { self, nixpkgs, lowdown-src }:

From c27fcd94ce3de918d506afe90759d97d1946eca1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Oct 2020 21:44:07 +0200
Subject: [PATCH 408/882] Remove buildStatic from checks

checks should be relatively fast, but buildStatic depends on a lot of
stuff that isn't in the binary cache (e.g. musl builds of Git and
Mercurial that we probably don't need since we don't link against
them...).
---
 flake.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 22ea2911e..abc614c8d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -440,8 +440,6 @@
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
-      } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) {
-        buildStatic = self.hydraJobs.buildStatic.${system};
       });
 
       packages = forAllSystems (system: {

From 9c3dc9d7ca11555fbafe77e9ca8ed9fc214ab2f8 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Mon, 7 Sep 2020 23:53:31 -0500
Subject: [PATCH 409/882] update macOS version handling for Big Sur

Keeping this commit narrow for reviewability, but some of these
conditionals will change in subsequent commits in this PR.

Fixes #3852.
---
 scripts/install-nix-from-closure.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 6efd8af18..e7e063007 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -24,9 +24,11 @@ fi
 
 # macOS support for 10.12.6 or higher
 if [ "$(uname -s)" = "Darwin" ]; then
-    macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
-    macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
-    if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then
+    IFS='.' read macos_major macos_minor macos_patch << EOF
+$(sw_vers -productVersion)
+EOF
+    if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then
+        # patch may not be present; command substitution for simplicity
         echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
         exit 1
     fi
@@ -88,7 +90,7 @@ while [ $# -gt 0 ]; do
             ) >&2
 
             # darwin and Catalina+
-            if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
+            if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then
                 (
                     echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
                     echo "              store and mount it at /nix. This is the recommended way to create"
@@ -109,7 +111,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
     fi
 
     info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
-    if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
+    if ! [ -e $dest ] && [ -n "$info" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then
         (
             echo ""
             echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."

From 1f02b65c590b5a33c1b70dba186eacfe3f67f149 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Tue, 8 Sep 2020 00:07:53 -0500
Subject: [PATCH 410/882] fix xpath and conditional bugs; xpath -> xmllint

- xpath -> xmllint: xpath's cli interface changed in Big Sur
  rather than add conditional logic for picking the correct
  syntax for xpath, I'm changing to xmllint --xpath, which
  appears to be consistent across versions I've tested...

- /plist/dict/key[text()='Writable']/following-sibling::true[1]
  doesn't do quite what's expected. It was written to try to
  select a <true /> node paired with the Writable key, but it
  will also select the *next* <true /> node that appears even
  if it was paired with another key.

- I think there's also a logic bug in the conditionals here.
  I'm not sure anyone ever actuall saw it, thanks to the xpath
  bug, though. With the xpath fix, this conditional passes if /nix
  does not exist, / IS writable, and the version is Catalina+.

  I think it meant to test for /nix does not exist, / is NOT
  writable, and the version is Catalina+. I reworked this lightly
  to make it a little clearer at the code level.
---
 scripts/install-nix-from-closure.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index e7e063007..4e64f3d43 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -110,8 +110,8 @@ if [ "$(uname -s)" = "Darwin" ]; then
         "$self/create-darwin-volume.sh"
     fi
 
-    info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
-    if ! [ -e $dest ] && [ -n "$info" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then
+    writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)"
+    if ! [ -e $dest ] && [ "$writable" = "false" ]; then
         (
             echo ""
             echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."

From e736f8f6e44180d7ed7cc1975b48c603c6c4f611 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Tue, 8 Sep 2020 00:45:27 -0500
Subject: [PATCH 411/882] replace xpath with xmllint --xpath; simplify

As mentioned in previous commit, Big Sur changes the syntax for the
xpath command slightly.

In the process of testing out replacements for these, I noticed a few
small simplification wins.
---
 scripts/create-darwin-volume.sh | 46 ++++++---------------------------
 1 file changed, 8 insertions(+), 38 deletions(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index dac30d72d..8c4558c7f 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -5,42 +5,13 @@ root_disk() {
     diskutil info -plist /
 }
 
-apfs_volumes_for() {
-    disk=$1
-    diskutil apfs list -plist "$disk"
-}
-
-disk_identifier() {
-    xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
-}
-
-volume_list_true() {
-    key=$1
-    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
-}
-
-volume_get_string() {
-    key=$1 i=$2
-    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
+# i.e., "disk1"
+root_disk_identifier() {
+    diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" -
 }
 
 find_nix_volume() {
-    disk=$1
-    i=1
-    volumes=$(apfs_volumes_for "$disk")
-    while true; do
-        name=$(echo "$volumes" | volume_get_string "Name" "$i")
-        if [ -z "$name" ]; then
-            break
-        fi
-        case "$name" in
-            [Nn]ix*)
-                echo "$name"
-                break
-                ;;
-        esac
-        i=$((i+1))
-    done
+    diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true
 }
 
 test_fstab() {
@@ -89,9 +60,7 @@ test_t2_chip_present(){
 }
 
 test_filevault_in_use() {
-    disk=$1
-    #      list vols on disk | get value of Filevault key | value is true
-    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
+    fdesetup isactive >/dev/null
 }
 
 # use after error msg for conditions we don't understand
@@ -143,12 +112,12 @@ main() {
         fi
     fi
 
-    disk=$(root_disk | disk_identifier)
+    disk="$(root_disk_identifier)"
     volume=$(find_nix_volume "$disk")
     if [ -z "$volume" ]; then
         echo "Creating a Nix Store volume..." >&2
 
-        if test_filevault_in_use "$disk"; then
+        if test_filevault_in_use; then
             # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
             # shows both filevault and encrypted at rest status, and it
             # may be the more semantic way to test for this? It'll show
@@ -178,6 +147,7 @@ main() {
     if ! test_fstab; then
         echo "Configuring /etc/fstab..." >&2
         label=$(echo "$volume" | sed 's/ /\\040/g')
+        # shellcheck disable=SC2209
         printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
     fi
 }

From fe807904e5e6e56b551f34f3586e69ea6498287c Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Tue, 8 Sep 2020 01:01:11 -0500
Subject: [PATCH 412/882] adapt to apfs.util flag diff in catalina/big sur

Fixes #3957. Just runs both forms to minimize moving parts.
---
 scripts/create-darwin-volume.sh | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index 8c4558c7f..32fa577a8 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -26,6 +26,20 @@ test_synthetic_conf() {
     grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
 }
 
+# Create the paths defined in synthetic.conf, saving us a reboot.
+create_synthetic_objects(){
+    # Big Sur takes away the -B flag we were using and replaces it
+    # with a -t flag that appears to do the same thing (but they
+    # don't behave exactly the same way in terms of return values).
+    # This feels a little dirty, but as far as I can tell the
+    # simplest way to get the right one is to just throw away stderr
+    # and call both... :]
+    {
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true # Big Sur
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true # Catalina
+    } >/dev/null 2>&1
+}
+
 test_nix() {
     test -d "/nix"
 }
@@ -101,7 +115,7 @@ main() {
 
     if ! test_nix; then
         echo "Creating mountpoint for /nix..." >&2
-        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
+        create_synthetic_objects # the ones we defined in synthetic.conf
         if ! test_nix; then
             sudo mkdir -p /nix 2>/dev/null || true
         fi

From 3a8699ac4ffc0d9b611c471e2668e6b22cc65767 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Thu, 10 Sep 2020 18:21:04 -0500
Subject: [PATCH 413/882] restore create-darwin-volume to release tarball

The move from release.nix to flake.nix appears to have lost some
changes from #3628 / 1c56f18a8122b605c28000e295d5e223f272cccd, leaving
create-darwin-volume.sh out of the release tarball.

Under the assumption that this was just an accident/byproduct of when
flake.nix split off and not intentional, I am restoring those edits.
---
 flake.nix | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index a50533a29..d0c3bfcd2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -235,6 +235,7 @@
             }
             ''
               cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+              cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
               substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
                 --subst-var-by nix ${nix} \
                 --subst-var-by cacert ${cacert}
@@ -253,6 +254,7 @@
                 # SC1090: Don't worry about not being able to find
                 #         $nix/etc/profile.d/nix.sh
                 shellcheck --exclude SC1090 $TMPDIR/install
+                shellcheck $TMPDIR/create-darwin-volume.sh
                 shellcheck $TMPDIR/install-darwin-multi-user.sh
                 shellcheck $TMPDIR/install-systemd-multi-user.sh
 
@@ -268,6 +270,7 @@
               fi
 
               chmod +x $TMPDIR/install
+              chmod +x $TMPDIR/create-darwin-volume.sh
               chmod +x $TMPDIR/install-darwin-multi-user.sh
               chmod +x $TMPDIR/install-systemd-multi-user.sh
               chmod +x $TMPDIR/install-multi-user
@@ -280,11 +283,15 @@
                 --absolute-names \
                 --hard-dereference \
                 --transform "s,$TMPDIR/install,$dir/install," \
+                --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
                 --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
                 --transform "s,$NIX_STORE,$dir/store,S" \
-                $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+                $TMPDIR/install \
+                $TMPDIR/create-darwin-volume.sh \
+                $TMPDIR/install-darwin-multi-user.sh \
                 $TMPDIR/install-systemd-multi-user.sh \
-                $TMPDIR/install-multi-user $TMPDIR/reginfo \
+                $TMPDIR/install-multi-user \
+                $TMPDIR/reginfo \
                 $(cat ${installerClosureInfo}/store-paths)
             '');
 

From b719f686a8c8936fe831ce730f28638d8b1e2982 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Fri, 11 Sep 2020 12:06:01 -0500
Subject: [PATCH 414/882] fix skipped multi-user install steps on macOS

Some of the changes in #3788 to support non-systemd Nix installs
don't appear to be aware that the darwin installer exists, which
resulted in some skipped steps and inappropriate instructions.
---
 scripts/install-darwin-multi-user.sh  |  7 +++++
 scripts/install-multi-user.sh         | 34 +++-----------------
 scripts/install-systemd-multi-user.sh | 45 ++++++++++++++++++++-------
 3 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 49076bd5c..a27be2a43 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -37,6 +37,13 @@ poly_service_setup_note() {
 EOF
 }
 
+poly_extra_try_me_commands(){
+  :
+}
+poly_extra_setup_instructions(){
+  :
+}
+
 poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
           cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index e5cc4d7ed..54edfe40d 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -71,11 +71,9 @@ uninstall_directions() {
     subheader "Uninstalling nix:"
     local step=0
 
-    if [ -e /run/systemd/system ] && poly_service_installed_check; then
+    if poly_service_installed_check; then
         step=$((step + 1))
         poly_service_uninstall_directions "$step"
-    else
-        step=$((step + 1))
     fi
 
     for profile_target in "${PROFILE_TARGETS[@]}"; do
@@ -255,40 +253,20 @@ function finish_success {
         echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
     fi
 
-    if [ -e /run/systemd/system ]; then
-        cat <<EOF
+    cat <<EOF
 
 Before Nix will work in your existing shells, you'll need to close
 them and open them again. Other than that, you should be ready to go.
 
 Try it! Open a new terminal, and type:
-
+$(poly_extra_try_me_commands)
   $ nix-shell -p nix-info --run "nix-info -m"
-
+$(poly_extra_setup_instructions)
 Thank you for using this installer. If you have any feedback, don't
 hesitate:
 
 $(contactme)
 EOF
-    else
-        cat <<EOF
-
-Before Nix will work in your existing shells, you'll need to close
-them and open them again. Other than that, you should be ready to go.
-
-Try it! Open a new terminal, and type:
-
-  $ sudo nix-daemon
-  $ nix-shell -p nix-info --run "nix-info -m"
-
-Additionally, you may want to add nix-daemon to your init-system.
-
-Thank you for using this installer. If you have any feedback, don't
-hesitate:
-
-$(contactme)
-EOF
-   fi
 
 }
 
@@ -725,9 +703,7 @@ main() {
     setup_default_profile
     place_nix_configuration
 
-    if [ -e /run/systemd/system ]; then
-        poly_configure_nix_daemon_service
-    fi
+    poly_configure_nix_daemon_service
 
     trap finish_success EXIT
 }
diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh
index e0201d53b..fda5ef600 100755
--- a/scripts/install-systemd-multi-user.sh
+++ b/scripts/install-systemd-multi-user.sh
@@ -72,24 +72,45 @@ poly_service_setup_note() {
 EOF
 }
 
+poly_extra_try_me_commands(){
+    if [ -e /run/systemd/system ]; then
+        :
+    else
+        cat <<EOF
+  $ sudo nix-daemon
+EOF
+    fi
+}
+poly_extra_setup_instructions(){
+    if [ -e /run/systemd/system ]; then
+        :
+    else
+        cat <<EOF
+Additionally, you may want to add nix-daemon to your init-system.
+
+EOF
+    fi
+}
+
 poly_configure_nix_daemon_service() {
-    _sudo "to set up the nix-daemon service" \
-          systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
+    if [ -e /run/systemd/system ]; then
+        _sudo "to set up the nix-daemon service" \
+              systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
 
-    _sudo "to set up the nix-daemon socket service" \
-          systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC"
+        _sudo "to set up the nix-daemon socket service" \
+              systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC"
 
-    handle_network_proxy
+        handle_network_proxy
 
-    _sudo "to load the systemd unit for nix-daemon" \
-          systemctl daemon-reload
+        _sudo "to load the systemd unit for nix-daemon" \
+              systemctl daemon-reload
 
-    _sudo "to start the nix-daemon.socket" \
-          systemctl start nix-daemon.socket
-
-    _sudo "to start the nix-daemon.service" \
-          systemctl restart nix-daemon.service
+        _sudo "to start the nix-daemon.socket" \
+              systemctl start nix-daemon.socket
 
+        _sudo "to start the nix-daemon.service" \
+              systemctl restart nix-daemon.service
+    fi
 }
 
 poly_group_exists() {

From c40bad415104398450866d32682011db0acb0310 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Fri, 11 Sep 2020 16:45:58 -0500
Subject: [PATCH 415/882] create missing profile files to fix zsh envvars

Env vars for ZSH were moved from /etc/zshrc to /etc/zshenv in #3608
to address an issue with zshrc getting clobbered by OS updates, but
/etc/zshenv doesn't exist by default--so *nothing* would get set up
for zsh users unless they already happened to have /etc/zshenv.

Creating these files if they don't exist. Also cut separate creation
of profile.d/nix.sh, which isn't needed now.
---
 scripts/install-multi-user.sh | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 54edfe40d..5e8b4ac18 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -608,24 +608,20 @@ EOF
 }
 
 configure_shell_profile() {
-    # If there is an /etc/profile.d directory, we want to ensure there
-    # is a nix.sh within it, so we can use the following loop to add
-    # the source lines to it. Note that I'm _not_ adding the source
-    # lines here, because we want to be using the regular machinery.
-    #
-    # If we go around that machinery, it becomes more complicated and
-    # adds complications to the uninstall instruction generator and
-    # old instruction sniffer as well.
-    if [ -d /etc/profile.d ]; then
-        _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \
-              touch /etc/profile.d/nix.sh
-    fi
-
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target" ]; then
             _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
                   cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
+        else
+            # try to create the file if its directory exists
+            target_dir="$(dirname "$profile_target")"
+            if [ -d "$target_dir" ]; then
+                _sudo "to create a stub $profile_target which will be updated" \
+                    touch "$profile_target"
+            fi
+        fi
 
+        if [ -e "$profile_target" ]; then
             shell_source_lines \
                 | _sudo "extend your $profile_target with nix-daemon settings" \
                         tee -a "$profile_target"

From f289bdb9d4e8432160c5dcdc037a930b2025d11b Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Mon, 19 Oct 2020 11:54:21 -0500
Subject: [PATCH 416/882] discourage casual Big Sur installs

---
 scripts/install-nix-from-closure.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 4e64f3d43..ea2e47b7f 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -27,6 +27,15 @@ if [ "$(uname -s)" = "Darwin" ]; then
     IFS='.' read macos_major macos_minor macos_patch << EOF
 $(sw_vers -productVersion)
 EOF
+    # TODO: this is a temporary speed-bump to keep people from naively installing Nix
+    # on macOS Big Sur (11.0+, 10.16+) until nixpkgs updates are ready for them.
+    # *Ideally* this is gone before next Nix release. If you're intentionally working on
+    # Nix + Big Sur, just comment out this block and be on your way :)
+    if [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 15 ]; }; then
+        echo "$0: nixpkgs isn't quite ready to support macOS $(sw_vers -productVersion) yet"
+        exit 1
+    fi
+
     if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then
         # patch may not be present; command substitution for simplicity
         echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"

From f6aaac2b5983bd87cd6e2f648c9dd835c6d9373c Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Tue, 20 Oct 2020 11:48:07 -0500
Subject: [PATCH 417/882] Make bash non-interactive for nix develop --phase

Fix #3975: Currently if Ctrl-C is pressed during a phase, the interactive subshell
is not exited. Removing --rcfile when --phase is present makes bash
non-interactive
---
 src/nix/develop.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 9372f43de..380417c82 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -368,7 +368,6 @@ struct CmdDevelop : Common, MixEnvironment
             // rid of that.
             script += fmt("foundMakefile=1\n");
             script += fmt("runHook %1%Phase\n", *phase);
-            script += fmt("exit 0\n", *phase);
         }
 
         else if (!command.empty()) {
@@ -408,7 +407,10 @@ struct CmdDevelop : Common, MixEnvironment
             ignoreException();
         }
 
-        auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
+        // If running a phase, don't want an interactive shell running after
+        // Ctrl-C, so don't pass --rcfile
+        auto args = phase ? Strings{std::string(baseNameOf(shell)), rcFilePath}
+            : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
 
         restoreAffinity();
         restoreSignals();

From 39fbd3d82823ae6eadfbf91bb2b356caef90841c Mon Sep 17 00:00:00 2001
From: Yuriy Taraday <yorik.sar@gmail.com>
Date: Wed, 21 Oct 2020 00:03:38 +0400
Subject: [PATCH 418/882] Fix iterating over $NIX_PROFILES in Zsh

NIX_PROFILES is space separated list of directories, and passing it into
for as is is considered to be 1-element list with the whole string. With
shwordsplit option Zsh emulates other shells in this regard ans
implicitely splits unquoted strings into words.

Fixes #4167.
---
 scripts/nix-profile-daemon.sh.in | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 4bc7c6fc5..500a98992 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -17,11 +17,21 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
     export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
 else
   # Fall back to what is in the nix profiles, favouring whatever is defined last.
-  for i in $NIX_PROFILES; do
-    if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
-      export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
+  check_nix_profiles() {
+    if [ "$ZSH_VERSION" ]; then
+      # Zsh by default doesn't split words in unquoted parameter expansion.
+      # Set local_options for these options to be reverted at the end of the function
+      # and shwordsplit to force splitting words in $NIX_PROFILES below.
+      setopt local_options shwordsplit
     fi
-  done
+    for i in $NIX_PROFILES; do
+      if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
+        export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
+      fi
+    done
+  }
+  check_nix_profiles
+  unset -f check_nix_profiles
 fi
 
 export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"

From 461cf2b85601e4510bff303e454059d80b9df8c0 Mon Sep 17 00:00:00 2001
From: Christian Kampka <christian@kampka.net>
Date: Mon, 19 Oct 2020 23:08:50 +0200
Subject: [PATCH 419/882] Add NIX_CONFIG env var for applying nix.conf
 overrides

---
 doc/manual/src/command-ref/conf-file-prefix.md | 3 +++
 doc/manual/src/command-ref/env-common.md       | 5 +++++
 src/libstore/globals.cc                        | 6 ++++++
 tests/config.sh                                | 9 +++++++++
 4 files changed, 23 insertions(+)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index 9987393d2..d38456788 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -19,6 +19,9 @@ By default Nix reads settings from the following places:
     and `XDG_CONFIG_HOME`. If these are unset, it will look in
     `$HOME/.config/nix.conf`.
 
+  - If `NIX_OPTIONS` is set, its contents is treated as the contents of
+    a configuration file.
+
 The configuration files consist of `name =
 value` pairs, one per line. Other files can be included with a line like
 `include
diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index 03016dba7..27e730fc8 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -81,6 +81,11 @@ Most Nix commands interpret the following environment variables:
     Overrides the location of the system Nix configuration directory
     (default `prefix/etc/nix`).
 
+  - `NIX_OPTIONS`  
+    Applies settings from Nix configuration from the environment.
+    The content is treated as if it was read from a Nix configuration file.
+    Settings are separated by the newline character.
+
   - `NIX_USER_CONF_FILES`  
     Overrides the location of the user Nix configuration files to load
     from (defaults to the XDG spec locations). The variable is treated
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 1238dc530..4df68d0c9 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -86,6 +86,12 @@ void loadConfFile()
     for (auto file = files.rbegin(); file != files.rend(); file++) {
         globalConfig.applyConfigFile(*file);
     }
+
+    auto nixConfEnv = getEnv("NIX_CONFIG");
+    if (nixConfEnv.has_value()) {
+        globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG");
+    }
+
 }
 
 std::vector<Path> getUserConfigFiles()
diff --git a/tests/config.sh b/tests/config.sh
index 8fa349f11..eaa46c395 100644
--- a/tests/config.sh
+++ b/tests/config.sh
@@ -16,3 +16,12 @@ here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
 export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf
 var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs)
 [[ $var == https://example.com ]]
+
+# Test that it's possible to load config from the environment
+prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
+export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes"
+exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
+exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
+[[ $prev != $exp_cores ]]
+[[ $exp_cores == "4242" ]]
+[[ $exp_features == "nix-command flakes" ]]
\ No newline at end of file

From bdf2bcc989348fdcf223c9e2a383618454453eb2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Oct 2020 19:18:02 +0200
Subject: [PATCH 420/882] Remove conf-file.xml

This was probably revived in a bad merge.
---
 doc/manual/command-ref/conf-file.xml | 1236 --------------------------
 1 file changed, 1236 deletions(-)
 delete mode 100644 doc/manual/command-ref/conf-file.xml

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
deleted file mode 100644
index d0f1b09ca..000000000
--- a/doc/manual/command-ref/conf-file.xml
+++ /dev/null
@@ -1,1236 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude"
-          xml:id="sec-conf-file"
-          version="5">
-
-<refmeta>
-  <refentrytitle>nix.conf</refentrytitle>
-  <manvolnum>5</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix.conf</refname>
-  <refpurpose>Nix configuration file</refpurpose>
-</refnamediv>
-
-<refsection><title>Description</title>
-
-<para>By default Nix reads settings from the following places:</para>
-
-<para>The system-wide configuration file
-<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-<filename>$NIX_CONF_DIR/nix.conf</filename> if
-<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
-client assumes that the daemon has already loaded them.
-</para>
-
-<para>User-specific configuration files:</para>
-
-<para>
-  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
-  <literal>:</literal> will be loaded in reverse order.
-</para>
-
-<para>
-  Otherwise it will look for <filename>nix/nix.conf</filename> files in
-  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
-
-  The default location is <filename>$HOME/.config/nix.conf</filename> if
-  those environment variables are unset.
-</para>
-
-<para>The configuration files consist of
-<literal><replaceable>name</replaceable> =
-<replaceable>value</replaceable></literal> pairs, one per line. Other
-files can be included with a line like <literal>include
-<replaceable>path</replaceable></literal>, where
-<replaceable>path</replaceable> is interpreted relative to the current
-conf file and a missing file is an error unless
-<literal>!include</literal> is used instead.
-Comments start with a <literal>#</literal> character.  Here is an
-example configuration file:</para>
-
-<programlisting>
-keep-outputs = true       # Nice for developers
-keep-derivations = true   # Idem
-</programlisting>
-
-<para>You can override settings on the command line using the
-<option>--option</option> flag, e.g. <literal>--option keep-outputs
-false</literal>.</para>
-
-<para>The following settings are currently available:
-
-<variablelist>
-
-
-  <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term>
-
-    <listitem>
-
-      <para>A list of URI prefixes to which access is allowed in
-      restricted evaluation mode. For example, when set to
-      <literal>https://github.com/NixOS</literal>, builtin functions
-      such as <function>fetchGit</function> are allowed to access
-      <literal>https://github.com/NixOS/patchelf.git</literal>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term>
-
-    <listitem><para>By default, Nix allows you to <function>import</function> from a derivation,
-    allowing building at evaluation time. With this option set to false, Nix will throw an error
-    when evaluating an expression that uses this feature, allowing users to ensure their evaluation
-    will not require any builds to take place.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term>
-
-    <listitem><para>(Linux-specific.) By default, builders on Linux
-    cannot acquire new privileges by calling setuid/setgid programs or
-    programs that have file capabilities. For example, programs such
-    as <command>sudo</command> or <command>ping</command> will
-    fail. (Note that in sandbox builds, no such programs are available
-    unless you bind-mount them into the sandbox via the
-    <option>sandbox-paths</option> option.) You can allow the
-    use of such programs by enabling this option. This is impure and
-    usually undesirable, but may be useful in certain scenarios
-    (e.g. to spin up containers or set up userspace network interfaces
-    in tests).</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term>
-
-    <listitem>
-
-      <para>A list of names of users (separated by whitespace) that
-      are allowed to connect to the Nix daemon. As with the
-      <option>trusted-users</option> option, you can specify groups by
-      prefixing them with <literal>@</literal>. Also, you can allow
-      all users by specifying <literal>*</literal>. The default is
-      <literal>*</literal>.</para>
-
-      <para>Note that trusted users are always allowed to connect.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix
-    automatically detects files in the store that have identical
-    contents, and replaces them with hard links to a single copy.
-    This saves disk space.  If set to <literal>false</literal> (the
-    default), you can still run <command>nix-store
-    --optimise</command> to get rid of duplicate
-    files.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-builders">
-    <term><literal>builders</literal></term>
-    <listitem>
-      <para>A list of machines on which to perform builds. <phrase
-      condition="manual">See <xref linkend="chap-distributed-builds"
-      /> for details.</phrase></para>
-    </listitem>
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-builders-use-substitutes"><term><literal>builders-use-substitutes</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix will instruct
-    remote build machines to use their own binary substitutes if available. In
-    practical terms, this means that remote hosts will fetch as many build
-    dependencies as possible from their own substitutes (e.g, from
-    <literal>cache.nixos.org</literal>), instead of waiting for this host to
-    upload them all. This can drastically reduce build times if the network
-    connection between this computer and the remote build host is slow. Defaults
-    to <literal>false</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term>
-
-    <listitem><para>This options specifies the Unix group containing
-    the Nix build user accounts.  In multi-user Nix installations,
-    builds should not be performed by the Nix account since that would
-    allow users to arbitrarily modify the Nix store and database by
-    supplying specially crafted builders; and they cannot be performed
-    by the calling user since that would allow him/her to influence
-    the build result.</para>
-
-    <para>Therefore, if this option is non-empty and specifies a valid
-    group, builds will be performed under the user accounts that are a
-    member of the group specified here (as listed in
-    <filename>/etc/group</filename>).  Those user accounts should not
-    be used for any other purpose!</para>
-
-    <para>Nix will never run two builds under the same user account at
-    the same time.  This is to prevent an obvious security hole: a
-    malicious user writing a Nix expression that modifies the build
-    result of a legitimate Nix expression being built by another user.
-    Therefore it is good to have as many Nix build user accounts as
-    you can spare.  (Remember: uids are cheap.)</para>
-
-    <para>The build users should have permission to create files in
-    the Nix store, but not delete them.  Therefore,
-    <filename>/nix/store</filename> should be owned by the Nix
-    account, its group should be the group specified here, and its
-    mode should be <literal>1775</literal>.</para>
-
-    <para>If the build users group is empty, builds will be performed
-    under the uid of the Nix process (that is, the uid of the caller
-    if <envar>NIX_REMOTE</envar> is empty, the uid under which the Nix
-    daemon runs if <envar>NIX_REMOTE</envar> is
-    <literal>daemon</literal>).  Obviously, this should not be used in
-    multi-user settings with untrusted users.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-compress-build-log"><term><literal>compress-build-log</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    build logs written to <filename>/nix/var/log/nix/drvs</filename>
-    will be compressed on the fly using bzip2.  Otherwise, they will
-    not be compressed.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
-
-    <listitem>
-
-      <para>The timeout (in seconds) for establishing connections in
-      the binary cache substituter.  It corresponds to
-      <command>curl</command>’s <option>--connect-timeout</option>
-      option.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term>
-
-    <listitem><para>Sets the value of the
-    <envar>NIX_BUILD_CORES</envar> environment variable in the
-    invocation of builders.  Builders can use this variable at their
-    discretion to control the maximum amount of parallelism.  For
-    instance, in Nixpkgs, if the derivation attribute
-    <varname>enableParallelBuilding</varname> is set to
-    <literal>true</literal>, the builder passes the
-    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
-    It can be overridden using the <option
-    linkend='opt-cores'>--cores</option> command line switch and
-    defaults to <literal>1</literal>.  The value <literal>0</literal>
-    means that the builder should use all available CPU cores in the
-    system.</para>
-
-    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term>
-  <listitem>
-    <para>
-      Absolute path to an executable capable of diffing build results.
-      The hook executes if <xref linkend="conf-run-diff-hook" /> is
-      true, and the output of a build is known to not be the same.
-      This program is not executed to determine if two results are the
-      same.
-    </para>
-
-    <para>
-      The diff hook is executed by the same user and group who ran the
-      build. However, the diff hook does not have write access to the
-      store path just built.
-    </para>
-
-    <para>The diff hook program receives three parameters:</para>
-
-    <orderedlist>
-      <listitem>
-        <para>
-          A path to the previous build's results
-        </para>
-      </listitem>
-
-      <listitem>
-        <para>
-          A path to the current build's results
-        </para>
-      </listitem>
-
-      <listitem>
-        <para>
-          The path to the build's derivation
-        </para>
-      </listitem>
-
-      <listitem>
-        <para>
-          The path to the build's scratch directory. This directory
-          will exist only if the build was run with
-          <option>--keep-failed</option>.
-        </para>
-      </listitem>
-    </orderedlist>
-
-    <para>
-      The stderr and stdout output from the diff hook will not be
-      displayed to the user. Instead, it will print to the nix-daemon's
-      log.
-    </para>
-
-    <para>When using the Nix daemon, <literal>diff-hook</literal> must
-    be set in the <filename>nix.conf</filename> configuration file, and
-    cannot be passed at the command line.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-enforce-determinism">
-    <term><literal>enforce-determinism</literal></term>
-
-    <listitem><para>See <xref linkend="conf-repeat" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-extra-sandbox-paths">
-    <term><literal>extra-sandbox-paths</literal></term>
-
-    <listitem><para>A list of additional paths appended to
-    <option>sandbox-paths</option>. Useful if you want to extend
-    its default value.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-extra-platforms"><term><literal>extra-platforms</literal></term>
-
-    <listitem><para>Platforms other than the native one which
-    this machine is capable of building for. This can be useful for
-    supporting additional architectures on compatible machines:
-    i686-linux can be built on x86_64-linux machines (and the default
-    for this setting reflects this); armv7 is backwards-compatible with
-    armv6 and armv5tel; some aarch64 machines can also natively run
-    32-bit ARM code; and qemu-user may be used to support non-native
-    platforms (though this may be slow and buggy). Most values for this
-    are not enabled by default because build systems will often
-    misdetect the target platform and generate incompatible code, so you
-    may wish to cross-check the results of using this option against
-    proper natively-built versions of your
-    derivations.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term>
-
-    <listitem><para>Additional binary caches appended to those
-    specified in <option>substituters</option>.  When used by
-    unprivileged users, untrusted substituters (i.e. those not listed
-    in <option>trusted-substituters</option>) are silently
-    ignored.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix will fall
-    back to building from source if a binary substitute fails.  This
-    is equivalent to the <option>--fallback</option> flag.  The
-    default is <literal>false</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, changes to the
-    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
-    synchronously flushed to disk.  This improves robustness in case
-    of system crashes, but reduces performance.  The default is
-    <literal>true</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
-
-    <listitem><para>A list of web servers used by
-    <function>builtins.fetchurl</function> to obtain files by hash.
-    Given a hash type <replaceable>ht</replaceable> and a base-16 hash
-    <replaceable>h</replaceable>, Nix will try to download the file
-    from
-    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
-    This allows files to be downloaded even if they have disappeared
-    from their original URI. For example, given the hashed mirror
-    <literal>http://tarballs.example.com/</literal>, when building the
-    derivation
-
-<programlisting>
-builtins.fetchurl {
-  url = "https://example.org/foo-1.2.3.tar.xz";
-  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-</programlisting>
-
-    Nix will attempt to download this file from
-    <literal>http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
-    first. If it is not available there, if will try the original URI.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
-
-    <listitem><para>The maximum number of parallel TCP connections
-    used to fetch files from binary caches and by other downloads. It
-    defaults to 25. 0 means no limit.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-keep-build-log"><term><literal>keep-build-log</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    Nix will write the build log of a derivation (i.e. the standard
-    output and error of its builder) to the directory
-    <filename>/nix/var/log/nix/drvs</filename>.  The build log can be
-    retrieved using the command <command>nix-store -l
-    <replaceable>path</replaceable></command>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term>
-
-    <listitem><para>If <literal>true</literal> (default), the garbage
-    collector will keep the derivations from which non-garbage store
-    paths were built.  If <literal>false</literal>, they will be
-    deleted unless explicitly registered as a root (or reachable from
-    other roots).</para>
-
-    <para>Keeping derivation around is useful for querying and
-    traceability (e.g., it allows you to ask with what dependencies or
-    options a store path was built), so by default this option is on.
-    Turn it off to save a bit of disk space (or a lot if
-    <literal>keep-outputs</literal> is also turned on).</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
-
-    <listitem><para>If <literal>false</literal> (default), derivations
-    are not stored in Nix user environments.  That is, the derivations of
-    any build-time-only dependencies may be garbage-collected.</para>
-
-    <para>If <literal>true</literal>, when you add a Nix derivation to
-    a user environment, the path of the derivation is stored in the
-    user environment.  Thus, the derivation will not be
-    garbage-collected until the user environment generation is deleted
-    (<command>nix-env --delete-generations</command>).  To prevent
-    build-time-only dependencies from being collected, you should also
-    turn on <literal>keep-outputs</literal>.</para>
-
-    <para>The difference between this option and
-    <literal>keep-derivations</literal> is that this one is
-    “sticky”: it applies to any user environment created while this
-    option was enabled, while <literal>keep-derivations</literal>
-    only applies at the moment the garbage collector is
-    run.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
-
-    <listitem><para>If <literal>true</literal>, the garbage collector
-    will keep the outputs of non-garbage derivations.  If
-    <literal>false</literal> (default), outputs will be deleted unless
-    they are GC roots themselves (or reachable from other roots).</para>
-
-    <para>In general, outputs must be registered as roots separately.
-    However, even if the output of a derivation is registered as a
-    root, the collector will still delete store paths that are used
-    only at build time (e.g., the C compiler, or source tarballs
-    downloaded from the network).  To prevent it from doing so, set
-    this option to <literal>true</literal>.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of bytes that a
-      builder can write to its stdout/stderr.  If the builder exceeds
-      this limit, it’s killed.  A value of <literal>0</literal> (the
-      default) means that there is no limit.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term>
-
-    <listitem><para>When a garbage collection is triggered by the
-    <literal>min-free</literal> option, it stops as soon as
-    <literal>max-free</literal> bytes are available. The default is
-    infinity (i.e. delete all garbage).</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
-
-    <listitem><para>This option defines the maximum number of jobs
-    that Nix will try to build in parallel.  The default is
-    <literal>1</literal>. The special value <literal>auto</literal>
-    causes Nix to use the number of CPUs in your system.  <literal>0</literal>
-    is useful when using remote builders to prevent any local builds (except for
-    <literal>preferLocalBuild</literal> derivation attribute which executes locally
-    regardless).  It can be
-    overridden using the <option
-    linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
-    command line switch.</para>
-
-    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of seconds that a
-      builder can go without producing any data on standard output or
-      standard error.  This is useful (for instance in an automated
-      build system) to catch builds that are stuck in an infinite
-      loop, or to catch remote builds that are hanging due to network
-      problems.  It can be overridden using the <option
-      linkend="opt-max-silent-time">--max-silent-time</option> command
-      line switch.</para>
-
-      <para>The value <literal>0</literal> means that there is no
-      timeout.  This is also the default.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term>
-
-    <listitem>
-      <para>When free disk space in <filename>/nix/store</filename>
-      drops below <literal>min-free</literal> during a build, Nix
-      performs a garbage-collection until <literal>max-free</literal>
-      bytes are available or there is no more garbage.  A value of
-      <literal>0</literal> (the default) disables this feature.</para>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-narinfo-cache-negative-ttl"><term><literal>narinfo-cache-negative-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for negative lookups. If a store path is
-      queried from a substituter but was not found, there will be a
-      negative lookup cached in the local disk cache database for the
-      specified duration.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-narinfo-cache-positive-ttl"><term><literal>narinfo-cache-positive-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for positive lookups. If a store path is
-      queried from a substituter, the result of the query will be cached
-      in the local disk cache database including some of the NAR
-      metadata. The default TTL is a month, setting a shorter TTL for
-      positive lookups can be useful for binary caches that have
-      frequent garbage collection, in which case having a more frequent
-      cache invalidation would prevent trying to pull the path again and
-      failing with a hash mismatch if the build isn't reproducible.
-      </para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
-
-    <listitem><para>If set to an absolute path to a <filename>netrc</filename>
-    file, Nix will use the HTTP authentication credentials in this file when
-    trying to download from a remote host through HTTP or HTTPS. Defaults to
-    <filename>$NIX_CONF_DIR/netrc</filename>.</para>
-
-    <para>The <filename>netrc</filename> file consists of a list of
-    accounts in the following format:
-
-<screen>
-machine <replaceable>my-machine</replaceable>
-login <replaceable>my-username</replaceable>
-password <replaceable>my-password</replaceable>
-</screen>
-
-    For the exact syntax, see <link
-    xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
-    <literal>curl</literal> documentation.</link></para>
-
-    <note><para>This must be an absolute path, and <literal>~</literal>
-    is not resolved. For example, <filename>~/.netrc</filename> won't
-    resolve to your home directory's <filename>.netrc</filename>.</para></note>
-    </listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-plugin-files">
-    <term><literal>plugin-files</literal></term>
-    <listitem>
-      <para>
-        A list of plugin files to be loaded by Nix. Each of these
-        files will be dlopened by Nix, allowing them to affect
-        execution through static initialization. In particular, these
-        plugins may construct static instances of RegisterPrimOp to
-        add new primops or constants to the expression language,
-        RegisterStoreImplementation to add new store implementations,
-        RegisterCommand to add new subcommands to the
-        <literal>nix</literal> command, and RegisterSetting to add new
-        nix config settings. See the constructors for those types for
-        more details.
-      </para>
-      <para>
-        Since these files are loaded into the same address space as
-        Nix itself, they must be DSOs compatible with the instance of
-        Nix running at the time (i.e. compiled against the same
-        headers, not linked to any incompatible libraries). They
-        should not be linked to any Nix libs directly, as those will
-        be available already at load time.
-      </para>
-      <para>
-        If an entry in the list is a directory, all files in the
-        directory are loaded as plugins (non-recursively).
-      </para>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
-
-    <listitem>
-
-
-      <para>If set, the path to a program that can set extra
-      derivation-specific settings for this system. This is used for settings
-      that can't be captured by the derivation model itself and are too variable
-      between different versions of the same system to be hard-coded into nix.
-      </para>
-
-      <para>The hook is passed the derivation path and, if sandboxes are enabled,
-      the sandbox directory. It can then modify the sandbox and send a series of
-      commands to modify various settings to stdout. The currently recognized
-      commands are:</para>
-
-      <variablelist>
-        <varlistentry xml:id="extra-sandbox-paths">
-          <term><literal>extra-sandbox-paths</literal></term>
-
-          <listitem>
-
-            <para>Pass a list of files and directories to be included in the
-            sandbox for this build. One entry per line, terminated by an empty
-            line. Entries have the same format as
-            <literal>sandbox-paths</literal>.</para>
-
-          </listitem>
-
-        </varlistentry>
-      </variablelist>
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-post-build-hook">
-    <term><literal>post-build-hook</literal></term>
-    <listitem>
-      <para>Optional. The path to a program to execute after each build.</para>
-
-      <para>This option is only settable in the global
-      <filename>nix.conf</filename>, or on the command line by trusted
-      users.</para>
-
-      <para>When using the nix-daemon, the daemon executes the hook as
-      <literal>root</literal>. If the nix-daemon is not involved, the
-      hook runs as the user executing the nix-build.</para>
-
-      <itemizedlist>
-        <listitem><para>The hook executes after an evaluation-time build.</para></listitem>
-        <listitem><para>The hook does not execute on substituted paths.</para></listitem>
-        <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
-        <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
-        <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem>
-      </itemizedlist>
-
-      <para>The program executes with no arguments. The program's environment
-      contains the following environment variables:</para>
-
-      <variablelist>
-        <varlistentry>
-          <term><envar>DRV_PATH</envar></term>
-          <listitem>
-            <para>The derivation for the built paths.</para>
-            <para>Example:
-            <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal>
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><envar>OUT_PATHS</envar></term>
-          <listitem>
-            <para>Output paths of the built derivation, separated by a space character.</para>
-            <para>Example:
-            <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
-            /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
-            /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
-            /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
-            /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-      </variablelist>
-
-      <para>See <xref linkend="chap-post-build-hook" /> for an example
-      implementation.</para>
-
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
-
-    <listitem><para>How many times to repeat builds to check whether
-    they are deterministic. The default value is 0. If the value is
-    non-zero, every build is repeated the specified number of
-    times. If the contents of any of the runs differs from the
-    previous ones and <xref linkend="conf-enforce-determinism" /> is
-    true, the build is rejected and the resulting store paths are not
-    registered as “valid” in Nix’s database.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    any non-content-addressed path added or copied to the Nix store
-    (e.g. when substituting from a binary cache) must have a valid
-    signature, that is, be signed using one of the keys listed in
-    <option>trusted-public-keys</option> or
-    <option>secret-key-files</option>. Set to <literal>false</literal>
-    to disable signature checking.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-restrict-eval"><term><literal>restrict-eval</literal></term>
-
-    <listitem>
-
-      <para>If set to <literal>true</literal>, the Nix evaluator will
-      not allow access to any files outside of the Nix search path (as
-      set via the <envar>NIX_PATH</envar> environment variable or the
-      <option>-I</option> option), or to URIs outside of
-      <option>allowed-uri</option>. The default is
-      <literal>false</literal>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-run-diff-hook"><term><literal>run-diff-hook</literal></term>
-  <listitem>
-    <para>
-      If true, enable the execution of <xref linkend="conf-diff-hook" />.
-    </para>
-
-    <para>
-      When using the Nix daemon, <literal>run-diff-hook</literal> must
-      be set in the <filename>nix.conf</filename> configuration file,
-      and cannot be passed at the command line.
-    </para>
-  </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, builds will be
-    performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
-    they’re isolated from the normal file system hierarchy and will
-    only see their dependencies in the Nix store, the temporary build
-    directory, private versions of <filename>/proc</filename>,
-    <filename>/dev</filename>, <filename>/dev/shm</filename> and
-    <filename>/dev/pts</filename> (on Linux), and the paths configured with the
-    <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal>
-    option</link>. This is useful to prevent undeclared dependencies
-    on files in directories such as <filename>/usr/bin</filename>. In
-    addition, on Linux, builds run in private PID, mount, network, IPC
-    and UTS namespaces to isolate them from other processes in the
-    system (except that fixed-output derivations do not run in private
-    network namespace to ensure they can access the network).</para>
-
-    <para>Currently, sandboxing only work on Linux and macOS. The use
-    of a sandbox requires that Nix is run as root (so you should use
-    the <link linkend='conf-build-users-group'>“build users”
-    feature</link> to perform the actual builds under different users
-    than root).</para>
-
-    <para>If this option is set to <literal>relaxed</literal>, then
-    fixed-output derivations and derivations that have the
-    <varname>__noChroot</varname> attribute set to
-    <literal>true</literal> do not run in sandboxes.</para>
-
-    <para>The default is <literal>true</literal> on Linux and
-    <literal>false</literal> on all other platforms.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term>
-
-    <listitem><para>This option determines the maximum size of the
-    <literal>tmpfs</literal> filesystem mounted on
-    <filename>/dev/shm</filename> in Linux sandboxes. For the format,
-    see the description of the <option>size</option> option of
-    <literal>tmpfs</literal> in
-    <citerefentry><refentrytitle>mount</refentrytitle><manvolnum>8</manvolnum></citerefentry>. The
-    default is <literal>50%</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-sandbox-paths">
-    <term><literal>sandbox-paths</literal></term>
-
-    <listitem><para>A list of paths bind-mounted into Nix sandbox
-    environments. You can use the syntax
-    <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal>
-    to mount a path in a different location in the sandbox; for
-    instance, <literal>/bin=/nix-bin</literal> will mount the path
-    <literal>/nix-bin</literal> as <literal>/bin</literal> inside the
-    sandbox. If <replaceable>source</replaceable> is followed by
-    <literal>?</literal>, then it is not an error if
-    <replaceable>source</replaceable> does not exist; for example,
-    <literal>/dev/nvidiactl?</literal> specifies that
-    <filename>/dev/nvidiactl</filename> will only be mounted in the
-    sandbox if it exists in the host filesystem.</para>
-
-    <para>Depending on how Nix was built, the default value for this option
-    may be empty or provide <filename>/bin/sh</filename> as a
-    bind-mount of <command>bash</command>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-secret-key-files"><term><literal>secret-key-files</literal></term>
-
-    <listitem><para>A whitespace-separated list of files containing
-    secret (private) keys. These are used to sign locally-built
-    paths. They can be generated using <command>nix-store
-    --generate-binary-cache-key</command>. The corresponding public
-    key can be distributed to other users, who can add it to
-    <option>trusted-public-keys</option> in their
-    <filename>nix.conf</filename>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term>
-
-    <listitem><para>Causes Nix to print out a stack trace in case of Nix
-    expression evaluation errors.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-substitute"><term><literal>substitute</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (default), Nix
-    will use binary substitutes if available.  This option can be
-    disabled to force building from source.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term>
-    <listitem>
-      <para>The timeout (in seconds) for receiving data from servers
-      during download. Nix cancels idle downloads after this timeout's
-      duration.</para>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
-
-    <listitem><para>A list of URLs of substituters, separated by
-    whitespace.  The default is
-    <literal>https://cache.nixos.org</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-system"><term><literal>system</literal></term>
-
-    <listitem><para>This option specifies the canonical Nix system
-    name of the current installation, such as
-    <literal>i686-linux</literal> or
-    <literal>x86_64-darwin</literal>.  Nix can only build derivations
-    whose <literal>system</literal> attribute equals the value
-    specified here.  In general, it never makes sense to modify this
-    value from its default, since you can use it to ‘lie’ about the
-    platform you are building on (e.g., perform a Mac OS build on a
-    Linux machine; the result would obviously be wrong).  It only
-    makes sense if the Nix binaries can run on multiple platforms,
-    e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and
-    <literal>i686-linux</literal>.</para>
-
-    <para>It defaults to the canonical Nix system name detected by
-    <filename>configure</filename> at build time.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term>
-
-    <listitem><para>A set of system “features” supported by this
-    machine, e.g. <literal>kvm</literal>. Derivations can express a
-    dependency on such features through the derivation attribute
-    <varname>requiredSystemFeatures</varname>. For example, the
-    attribute
-
-<programlisting>
-requiredSystemFeatures = [ "kvm" ];
-</programlisting>
-
-    ensures that the derivation can only be built on a machine with
-    the <literal>kvm</literal> feature.</para>
-
-    <para>This setting by default includes <literal>kvm</literal> if
-    <filename>/dev/kvm</filename> is accessible, and the
-    pseudo-features <literal>nixos-test</literal>,
-    <literal>benchmark</literal> and <literal>big-parallel</literal>
-    that are used in Nixpkgs to route builds to specific
-    machines.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-tarball-ttl"><term><literal>tarball-ttl</literal></term>
-
-    <listitem>
-      <para>Default: <literal>3600</literal> seconds.</para>
-
-      <para>The number of seconds a downloaded tarball is considered
-      fresh. If the cached tarball is stale, Nix will check whether
-      it is still up to date using the ETag header. Nix will download
-      a new version if the ETag header is unsupported, or the
-      cached ETag doesn't match.
-      </para>
-
-      <para>Setting the TTL to <literal>0</literal> forces Nix to always
-      check if the tarball is up to date.</para>
-
-      <para>Nix caches tarballs in
-      <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para>
-
-      <para>Files fetched via <envar>NIX_PATH</envar>,
-      <function>fetchGit</function>, <function>fetchMercurial</function>,
-      <function>fetchTarball</function>, and <function>fetchurl</function>
-      respect this TTL.
-      </para>
-    </listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of seconds that a
-      builder can run.  This is useful (for instance in an automated
-      build system) to catch builds that are stuck in an infinite loop
-      but keep writing to their standard output or standard error.  It
-      can be overridden using the <option
-      linkend="opt-timeout">--timeout</option> command line
-      switch.</para>
-
-      <para>The value <literal>0</literal> means that there is no
-      timeout.  This is also the default.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term>
-
-    <listitem>
-
-      <para>Default: <literal>false</literal>.</para>
-
-      <para>If set to <literal>true</literal>, the Nix evaluator will
-      trace every function call. Nix will print a log message at the
-      "vomit" level for every function entrance and function exit.</para>
-
-      <informalexample><screen>
-function-trace entered undefined position at 1565795816999559622
-function-trace exited undefined position at 1565795816999581277
-function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
-function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
-</screen></informalexample>
-
-      <para>The <literal>undefined position</literal> means the function
-      call is a builtin.</para>
-
-      <para>Use the <literal>contrib/stack-collapse.py</literal> script
-      distributed with the Nix source code to convert the trace logs
-      in to a format suitable for <command>flamegraph.pl</command>.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
-
-    <listitem><para>A whitespace-separated list of public keys. When
-    paths are copied from another Nix store (such as a binary cache),
-    they must be signed with one of these keys. For example:
-    <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term>
-
-    <listitem><para>A list of URLs of substituters, separated by
-    whitespace.  These are not used by default, but can be enabled by
-    users of the Nix daemon by specifying <literal>--option
-    substituters <replaceable>urls</replaceable></literal> on the
-    command line.  Unprivileged users are only allowed to pass a
-    subset of the URLs listed in <literal>substituters</literal> and
-    <literal>trusted-substituters</literal>.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
-
-    <listitem>
-
-      <para>A list of names of users (separated by whitespace) that
-      have additional rights when connecting to the Nix daemon, such
-      as the ability to specify additional binary caches, or to import
-      unsigned NARs. You can also specify groups by prefixing them
-      with <literal>@</literal>; for instance,
-      <literal>@wheel</literal> means all users in the
-      <literal>wheel</literal> group. The default is
-      <literal>root</literal>.</para>
-
-      <warning><para>Adding a user to <option>trusted-users</option>
-      is essentially equivalent to giving that user root access to the
-      system. For example, the user can set
-      <option>sandbox-paths</option> and thereby obtain read access to
-      directories that are otherwise inacessible to
-      them.</para></warning>
-
-    </listitem>
-
-  </varlistentry>
-
-</variablelist>
-</para>
-
-<refsection>
-  <title>Deprecated Settings</title>
-
-<para>
-
-<variablelist>
-
-  <varlistentry xml:id="conf-binary-caches">
-    <term><literal>binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>binary-caches</literal> is now an alias to
-    <xref linkend="conf-substituters" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-binary-cache-public-keys">
-    <term><literal>binary-cache-public-keys</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>binary-cache-public-keys</literal> is now an alias to
-    <xref linkend="conf-trusted-public-keys" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-compress-log">
-    <term><literal>build-compress-log</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-compress-log</literal> is now an alias to
-    <xref linkend="conf-compress-build-log" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-cores">
-    <term><literal>build-cores</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-cores</literal> is now an alias to
-    <xref linkend="conf-cores" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-extra-chroot-dirs">
-    <term><literal>build-extra-chroot-dirs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-extra-chroot-dirs</literal> is now an alias to
-    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-extra-sandbox-paths">
-    <term><literal>build-extra-sandbox-paths</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-extra-sandbox-paths</literal> is now an alias to
-    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-fallback">
-    <term><literal>build-fallback</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-fallback</literal> is now an alias to
-    <xref linkend="conf-fallback" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-jobs">
-    <term><literal>build-max-jobs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-jobs</literal> is now an alias to
-    <xref linkend="conf-max-jobs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-log-size">
-    <term><literal>build-max-log-size</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-log-size</literal> is now an alias to
-    <xref linkend="conf-max-build-log-size" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-max-silent-time">
-    <term><literal>build-max-silent-time</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-max-silent-time</literal> is now an alias to
-    <xref linkend="conf-max-silent-time" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-repeat">
-    <term><literal>build-repeat</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-repeat</literal> is now an alias to
-    <xref linkend="conf-repeat" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-timeout">
-    <term><literal>build-timeout</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-timeout</literal> is now an alias to
-    <xref linkend="conf-timeout" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-chroot">
-    <term><literal>build-use-chroot</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-chroot</literal> is now an alias to
-    <xref linkend="conf-sandbox" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-sandbox">
-    <term><literal>build-use-sandbox</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-sandbox</literal> is now an alias to
-    <xref linkend="conf-sandbox" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-build-use-substitutes">
-    <term><literal>build-use-substitutes</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>build-use-substitutes</literal> is now an alias to
-    <xref linkend="conf-substitute" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-gc-keep-derivations">
-    <term><literal>gc-keep-derivations</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>gc-keep-derivations</literal> is now an alias to
-    <xref linkend="conf-keep-derivations" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-gc-keep-outputs">
-    <term><literal>gc-keep-outputs</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>gc-keep-outputs</literal> is now an alias to
-    <xref linkend="conf-keep-outputs" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-env-keep-derivations">
-    <term><literal>env-keep-derivations</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>env-keep-derivations</literal> is now an alias to
-    <xref linkend="conf-keep-env-derivations" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-extra-binary-caches">
-    <term><literal>extra-binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>extra-binary-caches</literal> is now an alias to
-    <xref linkend="conf-extra-substituters" />.</para></listitem>
-  </varlistentry>
-
-  <varlistentry xml:id="conf-trusted-binary-caches">
-    <term><literal>trusted-binary-caches</literal></term>
-
-    <listitem><para><emphasis>Deprecated:</emphasis>
-    <literal>trusted-binary-caches</literal> is now an alias to
-    <xref linkend="conf-trusted-substituters" />.</para></listitem>
-  </varlistentry>
-</variablelist>
-</para>
-</refsection>
-
-</refsection>
-
-</refentry>

From 7f60f48e1aed14680eaf05154abf243ea764bf85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= <josef.kemetmueller@gmail.com>
Date: Wed, 21 Oct 2020 17:04:36 +0200
Subject: [PATCH 421/882] Fix the docs about the new NIX_CONFIG env var

This was accidentally documented as NIX_OPTIONS.
---
 doc/manual/src/command-ref/conf-file-prefix.md | 2 +-
 doc/manual/src/command-ref/env-common.md       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index d38456788..e53a3ddf4 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -19,7 +19,7 @@ By default Nix reads settings from the following places:
     and `XDG_CONFIG_HOME`. If these are unset, it will look in
     `$HOME/.config/nix.conf`.
 
-  - If `NIX_OPTIONS` is set, its contents is treated as the contents of
+  - If `NIX_CONFIG` is set, its contents is treated as the contents of
     a configuration file.
 
 The configuration files consist of `name =
diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index 27e730fc8..c670d82b8 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -81,7 +81,7 @@ Most Nix commands interpret the following environment variables:
     Overrides the location of the system Nix configuration directory
     (default `prefix/etc/nix`).
 
-  - `NIX_OPTIONS`  
+  - `NIX_CONFIG`  
     Applies settings from Nix configuration from the environment.
     The content is treated as if it was read from a Nix configuration file.
     Settings are separated by the newline character.

From e556a1beb73f5d5608d7a8bad0a0c13fd148ed0a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Oct 2020 17:54:21 +0200
Subject: [PATCH 422/882] nix develop: Handle 'declare -ax' in bash output

Fixes 'nix develop nixpkgs#qpdfview'.
---
 src/nix/develop.cc | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 380417c82..32669414b 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -39,21 +39,24 @@ BuildEnvironment readEnvironment(const Path & path)
     static std::string varNameRegex =
         R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
 
-    static std::regex declareRegex(
-        "^declare -x (" + varNameRegex + ")" +
-        R"re((?:="((?:[^"\\]|\\.)*)")?\n)re");
-
     static std::string simpleStringRegex =
         R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
 
-    static std::string quotedStringRegex =
+    static std::string dquotedStringRegex =
+        R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
+
+    static std::string squotedStringRegex =
         R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
 
     static std::string indexedArrayRegex =
         R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
 
+    static std::regex declareRegex(
+        "^declare -a?x (" + varNameRegex + ")(=(" +
+        dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
+
     static std::regex varRegex(
-        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
+        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
 
     /* Note: we distinguish between an indexed and associative array
        using the space before the closing parenthesis. Will

From f9438fb64a223c05ebcfffa9706e1ca811a87d70 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Oct 2020 21:06:36 +0200
Subject: [PATCH 423/882] nix develop: Add --redirect flag to redirect
 dependencies

This is primarily useful if you're hacking simultaneously on a package
and one of its dependencies. E.g. if you're hacking on Hydra and Nix,
you would start a dev shell for Nix, and then a dev shell for Hydra as
follows:

  $ nix develop \
    --redirect .#hydraJobs.build.x86_64-linux.nix ~/Dev/nix/outputs/out \
    --redirect .#hydraJobs.build.x86_64-linux.nix.dev ~/Dev/nix/outputs/dev

(This assumes hydraJobs.build.x86_64-linux has a passthru.nix
attribute. You can also use a store path.)

This causes all references in the environment to those store paths to
be rewritten to ~/Dev/nix/outputs/{out,dev}. Note: unfortunately, you
may need to set LD_LIBRARY_PATH=~/Dev/nix/outputs/out/lib because
Nixpkgs' ld-wrapper only adds -rpath entries for -L flags that point
to the Nix store.
---
 src/nix/develop.cc | 53 +++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 32669414b..7a5f7e218 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -185,7 +185,22 @@ struct Common : InstallableCommand, MixProfile
         "UID",
     };
 
+    std::vector<std::pair<std::string, std::string>> redirects;
+
+    Common()
+    {
+        addFlag({
+            .longName = "redirect",
+            .description = "redirect a store path to a mutable location",
+            .labels = {"installable", "outputs-dir"},
+            .handler = {[&](std::string installable, std::string outputsDir) {
+                redirects.push_back({installable, outputsDir});
+            }}
+        });
+    }
+
     std::string makeRcScript(
+        ref<Store> store,
         const BuildEnvironment & buildEnvironment,
         const Path & outputsDir = absPath(".") + "/outputs")
     {
@@ -217,6 +232,8 @@ struct Common : InstallableCommand, MixProfile
 
         out << "eval \"$shellHook\"\n";
 
+        auto script = out.str();
+
         /* Substitute occurrences of output paths. */
         auto outputs = buildEnvironment.env.find("outputs");
         assert(outputs != buildEnvironment.env.end());
@@ -230,7 +247,33 @@ struct Common : InstallableCommand, MixProfile
             rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
         }
 
-        return rewriteStrings(out.str(), rewrites);
+        /* Substitute redirects. */
+        for (auto & [installableS, dir] : redirects) {
+            dir = absPath(dir);
+            auto installable = parseInstallable(store, installableS);
+            auto buildable = installable->toBuildable();
+            auto doRedirect = [&](const StorePath & path)
+            {
+                auto from = store->printStorePath(path);
+                if (script.find(from) == std::string::npos)
+                    warn("'%s' (path '%s') is not used by this build environment", installable->what(), from);
+                else {
+                    printInfo("redirecting '%s' to '%s'", from, dir);
+                    rewrites.insert({from, dir});
+                }
+            };
+            std::visit(overloaded {
+                [&](const BuildableOpaque & bo) {
+                    doRedirect(bo.path);
+                },
+                [&](const BuildableFromDrv & bfd) {
+                    for (auto & [outputName, path] : bfd.outputs)
+                        if (path) doRedirect(*path);
+                },
+            }, buildable);
+        }
+
+        return rewriteStrings(script, rewrites);
     }
 
     Strings getDefaultFlakeAttrPaths() override
@@ -348,6 +391,10 @@ struct CmdDevelop : Common, MixEnvironment
                 "To use a build environment previously recorded in a profile:",
                 "nix develop /tmp/my-shell"
             },
+            Example{
+                "To replace all occurences of a store path with a writable directory:",
+                "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev"
+            },
         };
     }
 
@@ -357,7 +404,7 @@ struct CmdDevelop : Common, MixEnvironment
 
         auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
 
-        auto script = makeRcScript(buildEnvironment);
+        auto script = makeRcScript(store, buildEnvironment);
 
         if (verbosity >= lvlDebug)
             script += "set -x\n";
@@ -449,7 +496,7 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        std::cout << makeRcScript(buildEnvironment);
+        std::cout << makeRcScript(store, buildEnvironment);
     }
 };
 

From 750ce500c221ecd4720a5b02e3f3cbb0bc05ef9d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Oct 2020 12:03:15 +0200
Subject: [PATCH 424/882] Fix clang build

---
 src/nix/develop.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 7a5f7e218..d3c4761a7 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -248,9 +248,9 @@ struct Common : InstallableCommand, MixProfile
         }
 
         /* Substitute redirects. */
-        for (auto & [installableS, dir] : redirects) {
-            dir = absPath(dir);
-            auto installable = parseInstallable(store, installableS);
+        for (auto & [installable_, dir_] : redirects) {
+            auto dir = absPath(dir_);
+            auto installable = parseInstallable(store, installable_);
             auto buildable = installable->toBuildable();
             auto doRedirect = [&](const StorePath & path)
             {

From c189cf7e3361e3c4caa989bde057439485b2c227 Mon Sep 17 00:00:00 2001
From: tnias <phil@grmr.de>
Date: Sun, 25 Oct 2020 23:16:53 +0100
Subject: [PATCH 425/882] Add sha512 to hashAlgo listings in manpages (#4186)

---
 doc/manual/src/command-ref/nix-hash.md         | 2 +-
 doc/manual/src/command-ref/nix-prefetch-url.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index 7ed82cdfc..de0459b9e 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -45,7 +45,7 @@ md5sum`.
 
   - `--type` *hashAlgo*  
     Use the specified cryptographic hash algorithm, which can be one of
-    `md5`, `sha1`, and `sha256`.
+    `md5`, `sha1`, `sha256`, and `sha512`.
 
   - `--to-base16`  
     Don’t hash anything, but convert the base-32 hash representation
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index 78c612cd4..59ab89b29 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -39,7 +39,7 @@ Nix store is also printed.
 
   - `--type` *hashAlgo*  
     Use the specified cryptographic hash algorithm, which can be one of
-    `md5`, `sha1`, and `sha256`.
+    `md5`, `sha1`, `sha256`, and `sha512`.
 
   - `--print-path`  
     Print the store path of the downloaded file on standard output.

From ac0e24f21b162f99c8e9437c710797820345a7a2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 11:28:42 +0100
Subject: [PATCH 426/882] Revert "Bump version to 3.0"

This reverts commit 189e6f5e1d949f50ab0b6e5acd25e230d206692d.

After some discussion, it seems better not to bump the major version
number since most of the new features since 2.3 are marked
experimental.
---
 .version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.version b/.version
index f398a2061..7208c2182 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-3.0
\ No newline at end of file
+2.4
\ No newline at end of file

From dc7d1322efbaa176bff38b1ad15eab6e11c83340 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 14:24:25 +0100
Subject: [PATCH 427/882] Make the prompt used in development shells
 configurable

---
 src/nix/develop.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index d3c4761a7..8fea7ee9c 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -11,6 +11,19 @@
 
 using namespace nix;
 
+struct DevelopSettings : Config
+{
+    Setting<std::string> bashPrompt{this, "", "bash-prompt",
+        "The bash prompt (`PS1`) in `nix develop` shells."};
+
+    Setting<std::string> bashPromptSuffix{this, "", "bash-prompt-suffix",
+        "Suffix appended to the `PS1` environment variable in `nix develop` shells."};
+};
+
+static DevelopSettings developSettings;
+
+static GlobalConfig::Register rDevelopSettings(&developSettings);
+
 struct Var
 {
     bool exported = true;
@@ -429,6 +442,10 @@ struct CmdDevelop : Common, MixEnvironment
 
         else {
             script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n";
+            if (developSettings.bashPrompt != "")
+                script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt));
+            if (developSettings.bashPromptSuffix != "")
+                script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix));
         }
 
         writeFull(rcFileFd.get(), script);

From 9d5e9ef0da89fe4fd02d7053ee28d79df3245325 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 16:58:58 +0100
Subject: [PATCH 428/882] Move Explicit

---
 src/libexpr/flake/lockfile.cc    |  3 ++-
 src/libexpr/primops/fetchTree.cc |  5 +++--
 src/libfetchers/attrs.hh         | 12 ------------
 src/libutil/types.hh             | 12 ++++++++++++
 src/nix/flake.cc                 |  6 +++---
 5 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index bb46e1bb4..a01a63611 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -34,7 +34,8 @@ LockedNode::LockedNode(const nlohmann::json & json)
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
 {
     if (!lockedRef.input.isImmutable())
-        throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
+        throw Error("lockfile contains mutable lock '%s'",
+            fetchers::attrsToJson(lockedRef.input.toAttrs()));
 }
 
 StorePath LockedNode::computeStorePath(Store & store) const
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 7cd4d0fbf..8d7ae4c14 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -43,7 +43,8 @@ void emitTreeAttrs(
     }
 
     if (input.getType() == "git")
-        mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
+        mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
+            fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
     if (auto revCount = input.getRevCount())
         mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
@@ -101,7 +102,7 @@ static void fetchTree(
             else if (attr.value->type == tString)
                 addURI(state, attrs, attr.name, attr.value->string.s);
             else if (attr.value->type == tBool)
-                attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
+                attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
             else if (attr.value->type == tInt)
                 attrs.emplace(attr.name, attr.value->integer);
             else
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index 4b4630c80..56bcdcfc8 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -8,18 +8,6 @@
 
 namespace nix::fetchers {
 
-/* Wrap bools to prevent string literals (i.e. 'char *') from being
-   cast to a bool in Attr. */
-template<typename T>
-struct Explicit {
-    T t;
-
-    bool operator ==(const Explicit<T> & other) const
-    {
-        return t == other.t;
-    }
-};
-
 typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
 typedef std::map<std::string, Attr> Attrs;
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 6c4c5ab74..9c85fef62 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -34,4 +34,16 @@ struct OnStartup
     OnStartup(T && t) { t(); }
 };
 
+/* Wrap bools to prevent string literals (i.e. 'char *') from being
+   cast to a bool in Attr. */
+template<typename T>
+struct Explicit {
+    T t;
+
+    bool operator ==(const Explicit<T> & other) const
+    {
+        return t == other.t;
+    }
+};
+
 }
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 43176d887..790e1ce95 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -82,11 +82,11 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
     if (flake.description)
         j["description"] = *flake.description;
     j["originalUrl"] = flake.originalRef.to_string();
-    j["original"] = attrsToJson(flake.originalRef.toAttrs());
+    j["original"] = fetchers::attrsToJson(flake.originalRef.toAttrs());
     j["resolvedUrl"] = flake.resolvedRef.to_string();
-    j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs());
+    j["resolved"] = fetchers::attrsToJson(flake.resolvedRef.toAttrs());
     j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
-    j["locked"] = attrsToJson(flake.lockedRef.toAttrs());
+    j["locked"] = fetchers::attrsToJson(flake.lockedRef.toAttrs());
     if (auto rev = flake.lockedRef.input.getRev())
         j["revision"] = rev->to_string(Base16, false);
     if (auto revCount = flake.lockedRef.input.getRevCount())

From 1e66d146a343cff8ea60fa24b1642762de27c787 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 17:59:32 +0100
Subject: [PATCH 429/882] Fix test

---
 tests/binary-cache.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index e14cf882e..6415c9302 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -271,5 +271,5 @@ rm $cacheDir/$(hashpart $docPath).narinfo
 
 nix-store --delete $outPath $docPath
 # -vvv is the level that logs during the loop
-timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \
+timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \
   --option trusted-binary-caches "file://$cacheDir"  --no-require-sigs

From b875b8f45c8d73c26e2cf13843fa25cc6762eebc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 17:59:36 +0100
Subject: [PATCH 430/882] Remove edition field

---
 src/libexpr/flake/flake.cc | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index bae4d65e5..ca3b185f9 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -196,11 +196,6 @@ static Flake getFlake(
 
     expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
 
-    auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
-
-    if (vInfo.attrs->get(sEdition))
-        warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
-
     if (auto description = vInfo.attrs->get(state.sDescription)) {
         expectType(state, tString, *description->value, *description->pos);
         flake.description = description->value->string.s;
@@ -229,8 +224,7 @@ static Flake getFlake(
         throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
 
     for (auto & attr : *vInfo.attrs) {
-        if (attr.name != sEdition &&
-            attr.name != state.sDescription &&
+        if (attr.name != state.sDescription &&
             attr.name != sInputs &&
             attr.name != sOutputs)
             throw Error("flake '%s' has an unsupported attribute '%s', at %s",

From 14aecbb288b71bbb79ddca638918318c9298200b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 20:36:46 +0100
Subject: [PATCH 431/882] BaseSetting<StringMap>::set(): Don't append to
 previous value

---
 src/libutil/config.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 521733025..eef01bde2 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -270,6 +270,7 @@ template<> std::string BaseSetting<StringSet>::to_string() const
 
 template<> void BaseSetting<StringMap>::set(const std::string & str)
 {
+    value.clear();
     auto kvpairs = tokenizeString<Strings>(str);
     for (auto & s : kvpairs)
     {

From 731edf0d9be27d1a64c9645595c7efba31dde2b1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 20:37:11 +0100
Subject: [PATCH 432/882] isTrivial(): Support trivial lists

---
 src/libexpr/eval.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d6366050c..4de87d647 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -208,7 +208,8 @@ bool Value::isTrivial() const
         && (type != tThunk
             || (dynamic_cast<ExprAttrs *>(thunk.expr)
                 && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
-            || dynamic_cast<ExprLambda *>(thunk.expr));
+            || dynamic_cast<ExprLambda *>(thunk.expr)
+            || dynamic_cast<ExprList *>(thunk.expr));
 }
 
 

From 343239fc8a1993f707a990c2cd54a41f1fa3de99 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Oct 2020 20:45:39 +0100
Subject: [PATCH 433/882] Allow nix.conf options to be set in flake.nix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This makes it possible to have per-project configuration in flake.nix,
e.g. binary caches and other stuff:

  nixConfig.bash-prompt-suffix = "ngi# ";
  nixConfig.substituters = [ "https://cache.ngi0.nixos.org/" ];
---
 src/libexpr/flake/flake.cc | 70 +++++++++++++++++++++++++++++++++++---
 src/libexpr/flake/flake.hh | 11 +++++-
 src/nix/installables.cc    |  5 ++-
 3 files changed, 80 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index ca3b185f9..bdcf63c21 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -71,11 +71,17 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     return {std::move(tree), resolvedRef, lockedRef};
 }
 
-static void expectType(EvalState & state, ValueType type,
-    Value & value, const Pos & pos)
+static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
 {
     if (value.type == tThunk && value.isTrivial())
         state.forceValue(value, pos);
+}
+
+
+static void expectType(EvalState & state, ValueType type,
+    Value & value, const Pos & pos)
+{
+    forceTrivialValue(state, value, pos);
     if (value.type != type)
         throw Error("expected %s but got %s at %s",
             showType(type), showType(value.type), pos);
@@ -114,7 +120,6 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, tString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
-                state.forceValue(*attr.value);
                 if (attr.value->type == tString)
                     attrs.emplace(attr.name, attr.value->string.s);
                 else
@@ -223,10 +228,41 @@ static Flake getFlake(
     } else
         throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
 
+    auto sNixConfig = state.symbols.create("nixConfig");
+
+    if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
+        expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos);
+
+        for (auto & option : *nixConfig->value->attrs) {
+            forceTrivialValue(state, *option.value, *option.pos);
+            if (option.value->type == tString)
+                flake.config.options.insert({option.name, state.forceStringNoCtx(*option.value, *option.pos)});
+            else if (option.value->type == tInt)
+                flake.config.options.insert({option.name, state.forceInt(*option.value, *option.pos)});
+            else if (option.value->type == tBool)
+                flake.config.options.insert({option.name, state.forceBool(*option.value, *option.pos)});
+            else if (option.value->isList()) {
+                std::vector<std::string> ss;
+                for (unsigned int n = 0; n < option.value->listSize(); ++n) {
+                    auto elem = option.value->listElems()[n];
+                    if (elem->type != tString)
+                        throw TypeError("list element in flake configuration option '%s' is %s while a string is expected",
+                            option.name, showType(*option.value));
+                    ss.push_back(state.forceStringNoCtx(*elem, *option.pos));
+                }
+                flake.config.options.insert({option.name, ss});
+            }
+            else
+                throw TypeError("flake configuration option '%s' is %s",
+                    option.name, showType(*option.value));
+        }
+    }
+
     for (auto & attr : *vInfo.attrs) {
         if (attr.name != state.sDescription &&
             attr.name != sInputs &&
-            attr.name != sOutputs)
+            attr.name != sOutputs &&
+            attr.name != sNixConfig)
             throw Error("flake '%s' has an unsupported attribute '%s', at %s",
                 lockedRef, attr.name, *attr.pos);
     }
@@ -599,4 +635,30 @@ Fingerprint LockedFlake::getFingerprint() const
 
 Flake::~Flake() { }
 
+void ConfigFile::apply()
+{
+    for (auto & [name, value] : options) {
+        // FIXME: support 'trusted-public-keys' (and other options), but make it TOFU.
+        if (name != "bash-prompt-suffix" &&
+            name != "bash-prompt" &&
+            name != "substituters" &&
+            name != "extra-substituters")
+        {
+            warn("ignoring untrusted flake configuration option '%s'", name);
+            continue;
+        }
+        // FIXME: Move into libutil/config.cc.
+        if (auto s = std::get_if<std::string>(&value))
+            globalConfig.set(name, *s);
+        else if (auto n = std::get_if<int64_t>(&value))
+            globalConfig.set(name, fmt("%d", n));
+        else if (auto b = std::get_if<Explicit<bool>>(&value))
+            globalConfig.set(name, b->t ? "true" : "false");
+        else if (auto ss = std::get_if<std::vector<std::string>>(&value))
+            globalConfig.set(name, concatStringsSep(" ", *ss)); // FIXME: evil
+        else
+            assert(false);
+    }
+}
+
 }
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index cf62c7741..7eebd9044 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -47,8 +47,16 @@ struct FlakeInput
     FlakeInputs overrides;
 };
 
-// The Flake structure is the main internal representation of a flake.nix file.
+struct ConfigFile
+{
+    using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
 
+    std::map<std::string, ConfigValue> options;
+
+    void apply();
+};
+
+/* The contents of a flake.nix file. */
 struct Flake
 {
     FlakeRef originalRef;   // the original flake specification (by the user)
@@ -57,6 +65,7 @@ struct Flake
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
+    ConfigFile config; // 'nixConfig' attribute
     ~Flake();
 };
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 7473c9758..fb264491a 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -533,8 +533,11 @@ InstallableFlake::getCursors(EvalState & state)
 
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
-    if (!_lockedFlake)
+    if (!_lockedFlake) {
         _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
+        _lockedFlake->flake.config.apply();
+        // FIXME: send new config to the daemon.
+    }
     return _lockedFlake;
 }
 

From c092fa4702215fdb61611c5dd28194401d056170 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 23 Sep 2020 16:30:42 +0200
Subject: [PATCH 434/882] Allow non-CA derivations to depend on CA derivations

---
 src/libexpr/primops.cc                | 39 +++++++++++-----
 src/libstore/build/derivation-goal.cc | 12 ++++-
 src/libstore/derivations.cc           | 65 ++++++++++++++++++++++++---
 src/libstore/derivations.hh           | 14 +++++-
 src/libstore/local-store.cc           |  4 +-
 src/nix/show-derivation.cc            |  1 +
 tests/content-addressed.nix           | 11 ++++-
 tests/content-addressed.sh            |  1 +
 8 files changed, 124 insertions(+), 23 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 2b304aab0..236433ef1 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1089,18 +1089,35 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         // Regular, non-CA derivation should always return a single hash and not
         // hash per output.
-        Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
+        auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
+        std::visit(overloaded {
+            [&](Hash h) {
+                for (auto & i : outputs) {
+                    auto outPath = state.store->makeOutputPath(i, h, drvName);
+                    drv.env[i] = state.store->printStorePath(outPath);
+                    drv.outputs.insert_or_assign(i,
+                        DerivationOutput {
+                            .output = DerivationOutputInputAddressed {
+                                .path = std::move(outPath),
+                            },
+                        });
+                }
+            },
+            [&](CaOutputHashes) {
+                // Shouldn't happen as the toplevel derivation is not CA.
+                assert(false);
+            },
+            [&](UnknownHashes) {
+                for (auto & i : outputs) {
+                    drv.outputs.insert_or_assign(i,
+                        DerivationOutput {
+                            .output = DerivationOutputDeferred{},
+                        });
+                }
+            },
+        },
+        hashModulo);
 
-        for (auto & i : outputs) {
-            auto outPath = state.store->makeOutputPath(i, h, drvName);
-            drv.env[i] = state.store->printStorePath(outPath);
-            drv.outputs.insert_or_assign(i,
-                DerivationOutput {
-                    .output = DerivationOutputInputAddressed {
-                        .path = std::move(outPath),
-                    },
-                });
-        }
     }
 
     /* Write the resulting term into the Nix store directory. */
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index db0c2bb6c..cc8737fd5 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -493,7 +493,8 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
+        if ((!fullDrv.inputDrvs.empty() &&
+             fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) {
             /* We are be able to resolve this derivation based on the
                now-known results of dependencies. If so, we become a stub goal
                aliasing that resolved derivation goal */
@@ -3166,6 +3167,15 @@ void DerivationGoal::registerOutputs()
             [&](DerivationOutputCAFloating dof) {
                 return newInfoFromCA(dof);
             },
+                [&](DerivationOutputDeferred) {
+                // No derivation should reach that point without having been
+                // rewritten first
+                assert(false);
+                // Ugly, but the compiler insists on having this return a value
+                // of type `ValidPathInfo` despite the `assert(false)`, so
+                // let's provide it
+                return *(ValidPathInfo*)0;
+            },
         }, output.output);
 
         /* Calculate where we'll move the output files. In the checking case we
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 07b4e772b..3b3a25391 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -21,6 +21,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
         [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
             return std::nullopt;
         },
+        [](DerivationOutputDeferred) -> std::optional<StorePath> {
+            return std::nullopt;
+        },
     }, output);
 }
 
@@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) {
     case DerivationType::InputAddressed: return false;
     case DerivationType::CAFixed: return true;
     case DerivationType::CAFloating: return true;
+    case DerivationType::DeferredInputAddressed: return false;
     };
     // Since enums can have non-variant values, but making a `default:` would
     // disable exhaustiveness warnings.
@@ -48,6 +52,7 @@ bool derivationIsFixed(DerivationType dt) {
     case DerivationType::InputAddressed: return false;
     case DerivationType::CAFixed: return true;
     case DerivationType::CAFloating: return false;
+    case DerivationType::DeferredInputAddressed: return false;
     };
     assert(false);
 }
@@ -57,6 +62,7 @@ bool derivationIsImpure(DerivationType dt) {
     case DerivationType::InputAddressed: return false;
     case DerivationType::CAFixed: return true;
     case DerivationType::CAFloating: return false;
+    case DerivationType::DeferredInputAddressed: return false;
     };
     assert(false);
 }
@@ -180,6 +186,11 @@ static DerivationOutput parseDerivationOutput(const Store & store,
             };
         }
     } else {
+        if (pathS == "") {
+            return DerivationOutput {
+                .output = DerivationOutputDeferred { }
+            };
+        }
         validatePath(pathS);
         return DerivationOutput {
             .output = DerivationOutputInputAddressed {
@@ -325,6 +336,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
                 s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                 s += ','; printUnquotedString(s, "");
             },
+            [&](DerivationOutputDeferred) {
+                s += ','; printUnquotedString(s, "");
+                s += ','; printUnquotedString(s, "");
+                s += ','; printUnquotedString(s, "");
+            }
         }, i.second.output);
         s += ')';
     }
@@ -389,7 +405,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
 
 DerivationType BasicDerivation::type() const
 {
-    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
+    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
     std::optional<HashType> floatingHashType;
     for (auto & i : outputs) {
         std::visit(overloaded {
@@ -408,22 +424,27 @@ DerivationType BasicDerivation::type() const
                         throw Error("All floating outputs must use the same hash type");
                 }
             },
+            [&](DerivationOutputDeferred _) {
+               deferredIAOutputs.insert(i.first);
+            },
         }, i.second.output);
     }
 
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
         throw Error("Must have at least one output");
-    } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+    } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
         return DerivationType::InputAddressed;
-    } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+    } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
         if (fixedCAOutputs.size() > 1)
             // FIXME: Experimental feature?
             throw Error("Only one fixed output is allowed for now");
         if (*fixedCAOutputs.begin() != "out")
             throw Error("Single fixed output must be named \"out\"");
         return DerivationType::CAFixed;
-    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
+    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
         return DerivationType::CAFloating;
+    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
+        return DerivationType::DeferredInputAddressed;
     } else {
         throw Error("Can't mix derivation output types");
     }
@@ -454,6 +475,8 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
     return h->second;
 }
 
+UnknownHashes unknownHashes;
+
 /* See the header for interface details. These are the implementation details.
 
    For fixed-output derivations, each hash in the map is not the
@@ -476,7 +499,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
     /* Return a fixed hash for fixed-output derivations. */
     switch (drv.type()) {
     case DerivationType::CAFloating:
-        throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
+        return unknownHashes;
     case DerivationType::CAFixed: {
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
@@ -491,12 +514,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
     }
     case DerivationType::InputAddressed:
         break;
+    case DerivationType::DeferredInputAddressed:
+        break;
     }
 
     /* For other derivations, replace the inputs paths with recursive
        calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & i : drv.inputDrvs) {
+        bool hasUnknownHash = false;
         const auto & res = pathDerivationModulo(store, i.first);
         std::visit(overloaded {
             // Regular non-CA derivation, replace derivation
@@ -514,7 +540,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
                         justOut);
                 }
             },
+            [&](UnknownHashes) {
+                hasUnknownHash = true;
+            },
         }, res);
+        if (hasUnknownHash) {
+            return unknownHashes;
+        }
     }
 
     return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@@ -620,6 +652,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
                     << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
                     << "";
             },
+            [&](DerivationOutputDeferred) {
+                out << ""
+                    << ""
+                    << "";
+            },
         }, i.second.output);
     }
     worker_proto::write(store, out, drv.inputSrcs);
@@ -645,7 +682,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
 }
 
 
-// N.B. Outputs are left unchanged
 static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
 
     debug("Rewriting the derivation");
@@ -666,6 +702,21 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
         newEnv.emplace(envName, envValue);
     }
     drv.env = newEnv;
+
+    auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
+    for (auto & [outputName, output] : drv.outputs) {
+        if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
+            Hash h = std::get<Hash>(hashModulo);
+            auto outPath = store.makeOutputPath(outputName, h, drv.name);
+            drv.env[outputName] = store.printStorePath(outPath);
+            output = DerivationOutput {
+                .output = DerivationOutputInputAddressed {
+                    .path = std::move(outPath),
+                },
+            };
+        }
+    }
+
 }
 
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 6d292b2e5..f9ba935e6 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -41,12 +41,18 @@ struct DerivationOutputCAFloating
     HashType hashType;
 };
 
+/* Input-addressed output which depends on a (CA) derivation whose hash isn't
+ * known atm
+ */
+struct DerivationOutputDeferred {};
+
 struct DerivationOutput
 {
     std::variant<
         DerivationOutputInputAddressed,
         DerivationOutputCAFixed,
-        DerivationOutputCAFloating
+        DerivationOutputCAFloating,
+        DerivationOutputDeferred
     > output;
     std::optional<HashType> hashAlgoOpt(const Store & store) const;
     /* Note, when you use this function you should make sure that you're passing
@@ -72,6 +78,7 @@ typedef std::map<string, string> StringPairs;
 
 enum struct DerivationType : uint8_t {
     InputAddressed,
+    DeferredInputAddressed,
     CAFixed,
     CAFloating,
 };
@@ -167,9 +174,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
 // whose output hashes are always known since they are fixed up-front.
 typedef std::map<std::string, Hash> CaOutputHashes;
 
+struct UnknownHashes {};
+
 typedef std::variant<
     Hash, // regular DRV normalized hash
-    CaOutputHashes
+    CaOutputHashes, // Fixed-output derivation hashes
+    UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies
 > DrvHashModulo;
 
 /* Returns hashes with the details of fixed-output subderivations
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d29236a9c..d29a68179 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -573,6 +573,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
             [&](DerivationOutputCAFloating _) {
                 /* Nothing to check */
             },
+            [&](DerivationOutputDeferred) {
+            },
         }, i.second.output);
     }
 }
@@ -817,7 +819,7 @@ std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivati
     }
     /* can't just use else-if instead of `!haveCached` because we need to unlock
        `drvPathResolutions` before it is locked in `Derivation::resolve`. */
-    if (!haveCached && drv.type() == DerivationType::CAFloating) {
+    if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) {
         /* Try resolve drv and use that path instead. */
         auto attempt = drv.tryResolve(*this);
         if (!attempt)
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 2542537d3..6d4f295d7 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -82,6 +82,7 @@ struct CmdShowDerivation : InstallablesCommand
                         [&](DerivationOutputCAFloating dof) {
                             outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                         },
+                        [&](DerivationOutputDeferred) {},
                     }, output.output);
                 }
             }
diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 3dcf916c3..8ca96d4bf 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -15,7 +15,7 @@ rec {
     '';
   };
   rootCA = mkDerivation {
-    name = "dependent";
+    name = "rootCA";
     outputs = [ "out" "dev" ];
     buildCommand = ''
       echo "building a CA derivation"
@@ -51,4 +51,13 @@ rec {
     outputHashMode = "recursive";
     outputHashAlgo = "sha256";
   };
+  dependentNonCA = mkDerivation {
+    name = "dependent-non-ca";
+    buildCommand = ''
+      echo "Didn't cut-off"
+      echo "building dependent-non-ca"
+      mkdir -p $out
+      echo ${rootCA}/non-ca-hello > $out/dep
+    '';
+  };
 }
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 61ec03fe3..547919660 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -22,6 +22,7 @@ secondSeedArgs=(-j0)
 # dependent derivations always being already built.
 #testDerivation dependentCA
 testDerivation transitivelyDependentCA
+testDerivation dependentNonCA
 
 nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
 nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true

From bc081bcd816542d66f1578788b93df4d7e07b135 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 24 Sep 2020 10:11:58 +0200
Subject: [PATCH 435/882] Inline `unkownHashes`

See https://github.com/NixOS/nix/pull/4056#discussion_r493661632
---
 src/libstore/derivations.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 3b3a25391..517ecfaa2 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -475,8 +475,6 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
     return h->second;
 }
 
-UnknownHashes unknownHashes;
-
 /* See the header for interface details. These are the implementation details.
 
    For fixed-output derivations, each hash in the map is not the
@@ -499,7 +497,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
     /* Return a fixed hash for fixed-output derivations. */
     switch (drv.type()) {
     case DerivationType::CAFloating:
-        return unknownHashes;
+        return UnknownHashes {};
     case DerivationType::CAFixed: {
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
@@ -545,7 +543,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
             },
         }, res);
         if (hasUnknownHash) {
-            return unknownHashes;
+            return UnknownHashes {};
         }
     }
 

From ab21ab65016275c224d1d40c42bdfed80dfbcbb0 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 7 Oct 2020 12:24:53 +0200
Subject: [PATCH 436/882] Test the remote caching of non-ca-depending-on-ca
 derivations

Although the non-resolved derivation will never get a cache-hit (it
doesn't have an output path to query the cache for anyways), we might
get one on the resolved derivation.
---
 tests/content-addressed.sh | 55 +++++++++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 15 deletions(-)

diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 547919660..bdab09c86 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -5,24 +5,49 @@ source common.sh
 drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1)
 nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
 
-testDerivation () {
+buildAttr () {
     local derivationPath=$1
-    local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
+    shift
+    local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
+    args+=("$@")
+    nix-build "${args[@]}"
+}
+
+testRemoteCache () {
+    clearCache
+    local outPath=$(buildAttr dependentNonCA)
+    nix copy --to file://$cacheDir $outPath
+    clearStore
+    buildAttr dependentNonCA --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca")
+}
+
+testDeterministicCA () {
+    [[ $(buildAttr rootCA) = $(buildAttr rootCA) ]]
+}
+
+testCutoffFor () {
     local out1 out2
-    out1=$(nix-build "${commonArgs[@]}" --arg seed 1)
-    out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${secondSeedArgs[@]}")
+    out1=$(buildAttr $1)
+    # The seed only changes the root derivation, and not it's output, so the
+    # dependent derivations should only need to be built once.
+    out2=$(buildAttr $1 -j0)
     test "$out1" == "$out2"
 }
 
-testDerivation rootCA
-# The seed only changes the root derivation, and not it's output, so the
-# dependent derivations should only need to be built once.
-secondSeedArgs=(-j0)
-# Don't directly build depenentCA, that way we'll make sure we dodn't rely on
-# dependent derivations always being already built.
-#testDerivation dependentCA
-testDerivation transitivelyDependentCA
-testDerivation dependentNonCA
+testCutoff () {
+    # Don't directly build depenentCA, that way we'll make sure we dodn't rely on
+    # dependent derivations always being already built.
+    #testDerivation dependentCA
+    testCutoffFor transitivelyDependentCA
+    testCutoffFor dependentNonCA
+}
 
-nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
-nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
+testGC () {
+    nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
+    nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
+}
+
+testRemoteCache
+testDeterministicCA
+testCutoff
+testGC

From 82e4d2a82ef11a95ff8e9aba3062db50145daab1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 28 Oct 2020 05:13:18 +0000
Subject: [PATCH 437/882] No x86_32 static nix jobs for now

Fixes #4175
---
 flake.nix | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index 0233962b4..2abbdff53 100644
--- a/flake.nix
+++ b/flake.nix
@@ -16,7 +16,8 @@
 
       officialRelease = false;
 
-      linuxSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
+      linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
+      linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
       systems = linuxSystems ++ [ "x86_64-darwin" ];
 
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
@@ -228,7 +229,7 @@
         # Binary package for various platforms.
         build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix);
 
-        buildStatic = nixpkgs.lib.genAttrs linuxSystems (system: self.packages.${system}.nix-static);
+        buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
@@ -451,7 +452,7 @@
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;
-      } // nixpkgs.lib.optionalAttrs (builtins.elem system linuxSystems) {
+      } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;
         in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {

From a5019f0508be961bf0230d2a528d30d3ded4b12a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Oct 2020 20:45:57 +0100
Subject: [PATCH 438/882] Consistency

---
 src/libutil/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 4a94f0dfd..8efff190a 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -192,7 +192,7 @@ Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType)
     // Either the string or user must provide the type, if they both do they
     // must agree.
     if (!optParsedType && !optType)
-        throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
+        throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest);
     else if (optParsedType && optType && *optParsedType != *optType)
         throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
 

From 6a4bf535d8126d8dc7306b3940bb49e0cc014a56 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Wed, 28 Oct 2020 09:41:18 -0500
Subject: [PATCH 439/882] Capitalize JSON for consistency

---
 src/libexpr/flake/lockfile.cc | 14 +++++++-------
 src/libexpr/flake/lockfile.hh |  2 +-
 src/libfetchers/attrs.cc      |  4 ++--
 src/libfetchers/attrs.hh      |  2 +-
 src/libfetchers/cache.cc      | 20 ++++++++++----------
 src/libfetchers/fetchers.cc   |  6 +++---
 src/libfetchers/registry.cc   |  6 +++---
 src/libmain/loggers.cc        |  4 ++--
 src/libmain/loggers.hh        |  2 +-
 src/libutil/logging.cc        |  2 +-
 src/nix/flake.cc              | 12 ++++++------
 11 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index a01a63611..8e2f7131f 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -35,7 +35,7 @@ LockedNode::LockedNode(const nlohmann::json & json)
 {
     if (!lockedRef.input.isImmutable())
         throw Error("lockfile contains mutable lock '%s'",
-            fetchers::attrsToJson(lockedRef.input.toAttrs()));
+            fetchers::attrsToJSON(lockedRef.input.toAttrs()));
 }
 
 StorePath LockedNode::computeStorePath(Store & store) const
@@ -111,7 +111,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
     // a bit since we don't need to worry about cycles.
 }
 
-nlohmann::json LockFile::toJson() const
+nlohmann::json LockFile::toJSON() const
 {
     nlohmann::json nodes;
     std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
@@ -155,8 +155,8 @@ nlohmann::json LockFile::toJson() const
         }
 
         if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
-            n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
-            n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
+            n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
+            n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
             if (!lockedNode->isFlake) n["flake"] = false;
         }
 
@@ -175,7 +175,7 @@ nlohmann::json LockFile::toJson() const
 
 std::string LockFile::to_string() const
 {
-    return toJson().dump(2);
+    return toJSON().dump(2);
 }
 
 LockFile LockFile::read(const Path & path)
@@ -186,7 +186,7 @@ LockFile LockFile::read(const Path & path)
 
 std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
 {
-    stream << lockFile.toJson().dump(2);
+    stream << lockFile.toJSON().dump(2);
     return stream;
 }
 
@@ -224,7 +224,7 @@ bool LockFile::isImmutable() const
 bool LockFile::operator ==(const LockFile & other) const
 {
     // FIXME: slow
-    return toJson() == other.toJson();
+    return toJSON() == other.toJSON();
 }
 
 InputPath parseInputPath(std::string_view s)
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
index 627794d8c..96f1edc76 100644
--- a/src/libexpr/flake/lockfile.hh
+++ b/src/libexpr/flake/lockfile.hh
@@ -52,7 +52,7 @@ struct LockFile
     LockFile() {};
     LockFile(const nlohmann::json & json, const Path & path);
 
-    nlohmann::json toJson() const;
+    nlohmann::json toJSON() const;
 
     std::string to_string() const;
 
diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index 1e59faa73..720b19fcd 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -23,7 +23,7 @@ Attrs jsonToAttrs(const nlohmann::json & json)
     return attrs;
 }
 
-nlohmann::json attrsToJson(const Attrs & attrs)
+nlohmann::json attrsToJSON(const Attrs & attrs)
 {
     nlohmann::json json;
     for (auto & attr : attrs) {
@@ -44,7 +44,7 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
     if (i == attrs.end()) return {};
     if (auto v = std::get_if<std::string>(&i->second))
         return *v;
-    throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
+    throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump());
 }
 
 std::string getStrAttr(const Attrs & attrs, const std::string & name)
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index 56bcdcfc8..a2d53a7bf 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -13,7 +13,7 @@ typedef std::map<std::string, Attr> Attrs;
 
 Attrs jsonToAttrs(const nlohmann::json & json);
 
-nlohmann::json attrsToJson(const Attrs & attrs);
+nlohmann::json attrsToJSON(const Attrs & attrs);
 
 std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
 
diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index e1c7f3dee..34ff6f85b 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -55,8 +55,8 @@ struct CacheImpl : Cache
         bool immutable) override
     {
         _state.lock()->add.use()
-            (attrsToJson(inAttrs).dump())
-            (attrsToJson(infoAttrs).dump())
+            (attrsToJSON(inAttrs).dump())
+            (attrsToJSON(infoAttrs).dump())
             (store->printStorePath(storePath))
             (immutable)
             (time(0)).exec();
@@ -70,7 +70,7 @@ struct CacheImpl : Cache
             if (!res->expired)
                 return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
             debug("ignoring expired cache entry '%s'",
-                attrsToJson(inAttrs).dump());
+                attrsToJSON(inAttrs).dump());
         }
         return {};
     }
@@ -81,15 +81,15 @@ struct CacheImpl : Cache
     {
         auto state(_state.lock());
 
-        auto inAttrsJson = attrsToJson(inAttrs).dump();
+        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
 
-        auto stmt(state->lookup.use()(inAttrsJson));
+        auto stmt(state->lookup.use()(inAttrsJSON));
         if (!stmt.next()) {
-            debug("did not find cache entry for '%s'", inAttrsJson);
+            debug("did not find cache entry for '%s'", inAttrsJSON);
             return {};
         }
 
-        auto infoJson = stmt.getStr(0);
+        auto infoJSON = stmt.getStr(0);
         auto storePath = store->parseStorePath(stmt.getStr(1));
         auto immutable = stmt.getInt(2) != 0;
         auto timestamp = stmt.getInt(3);
@@ -97,16 +97,16 @@ struct CacheImpl : Cache
         store->addTempRoot(storePath);
         if (!store->isValidPath(storePath)) {
             // FIXME: we could try to substitute 'storePath'.
-            debug("ignoring disappeared cache entry '%s'", inAttrsJson);
+            debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
             return {};
         }
 
         debug("using cache entry '%s' -> '%s', '%s'",
-            inAttrsJson, infoJson, store->printStorePath(storePath));
+            inAttrsJSON, infoJSON, store->printStorePath(storePath));
 
         return Result {
             .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
-            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
             .storePath = std::move(storePath)
         };
     }
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 49851f7bc..e6741a451 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -65,7 +65,7 @@ Input Input::fromAttrs(Attrs && attrs)
 ParsedURL Input::toURL() const
 {
     if (!scheme)
-        throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
+        throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs));
     return scheme->toURL(*this);
 }
 
@@ -110,7 +110,7 @@ bool Input::contains(const Input & other) const
 std::pair<Tree, Input> Input::fetch(ref<Store> store) const
 {
     if (!scheme)
-        throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
+        throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
 
     /* The tree may already be in the Nix store, or it could be
        substituted (which is often faster than fetching from the
@@ -247,7 +247,7 @@ std::optional<time_t> Input::getLastModified() const
 
 ParsedURL InputScheme::toURL(const Input & input)
 {
-    throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
+    throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs));
 }
 
 Input InputScheme::applyOverrides(
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 2426882ca..81b2227de 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -60,10 +60,10 @@ void Registry::write(const Path & path)
     nlohmann::json arr;
     for (auto & entry : entries) {
         nlohmann::json obj;
-        obj["from"] = attrsToJson(entry.from.toAttrs());
-        obj["to"] = attrsToJson(entry.to.toAttrs());
+        obj["from"] = attrsToJSON(entry.from.toAttrs());
+        obj["to"] = attrsToJSON(entry.to.toAttrs());
         if (!entry.extraAttrs.empty())
-            obj["to"].update(attrsToJson(entry.extraAttrs));
+            obj["to"].update(attrsToJSON(entry.extraAttrs));
         if (entry.exact)
             obj["exact"] = true;
         arr.emplace_back(std::move(obj));
diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index 0a7291780..cdf23859b 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -12,7 +12,7 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
     else if (logFormatStr == "raw-with-logs")
         return LogFormat::rawWithLogs;
     else if (logFormatStr == "internal-json")
-        return LogFormat::internalJson;
+        return LogFormat::internalJSON;
     else if (logFormatStr == "bar")
         return LogFormat::bar;
     else if (logFormatStr == "bar-with-logs")
@@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
         return makeSimpleLogger(false);
     case LogFormat::rawWithLogs:
         return makeSimpleLogger(true);
-    case LogFormat::internalJson:
+    case LogFormat::internalJSON:
         return makeJSONLogger(*makeSimpleLogger(true));
     case LogFormat::bar:
         return makeProgressBar();
diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh
index cada03110..f3c759193 100644
--- a/src/libmain/loggers.hh
+++ b/src/libmain/loggers.hh
@@ -7,7 +7,7 @@ namespace nix {
 enum class LogFormat {
   raw,
   rawWithLogs,
-  internalJson,
+  internalJSON,
   bar,
   barWithLogs,
 };
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 8a6752e22..6fd0dacef 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -306,7 +306,7 @@ bool handleJSONLogMessage(const std::string & msg,
 
     } catch (std::exception & e) {
         logError({
-            .name = "Json log message",
+            .name = "JSON log message",
             .hint = hintfmt("bad log message from builder: %s", e.what())
         });
     }
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 790e1ce95..7a7c71676 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -76,17 +76,17 @@ static void printFlakeInfo(const Store & store, const Flake & flake)
             std::put_time(std::localtime(&*lastModified), "%F %T"));
 }
 
-static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
+static nlohmann::json flakeToJSON(const Store & store, const Flake & flake)
 {
     nlohmann::json j;
     if (flake.description)
         j["description"] = *flake.description;
     j["originalUrl"] = flake.originalRef.to_string();
-    j["original"] = fetchers::attrsToJson(flake.originalRef.toAttrs());
+    j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs());
     j["resolvedUrl"] = flake.resolvedRef.to_string();
-    j["resolved"] = fetchers::attrsToJson(flake.resolvedRef.toAttrs());
+    j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs());
     j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
-    j["locked"] = fetchers::attrsToJson(flake.lockedRef.toAttrs());
+    j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs());
     if (auto rev = flake.lockedRef.input.getRev())
         j["revision"] = rev->to_string(Base16, false);
     if (auto revCount = flake.lockedRef.input.getRevCount())
@@ -139,7 +139,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
         auto flake = getFlake();
 
         if (json) {
-            auto json = flakeToJson(*store, flake);
+            auto json = flakeToJSON(*store, flake);
             logger->cout("%s", json.dump());
         } else
             printFlakeInfo(*store, flake);
@@ -158,7 +158,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
         auto flake = lockFlake();
 
         if (json)
-            logger->cout("%s", flake.lockFile.toJson());
+            logger->cout("%s", flake.lockFile.toJSON());
         else {
             logger->cout("%s", flake.flake.lockedRef);
 

From 869c0321ff4829f56e10316f401a0f9268c1a8b0 Mon Sep 17 00:00:00 2001
From: stev <stefan.jaax@gmail.com>
Date: Thu, 29 Oct 2020 00:33:14 +0100
Subject: [PATCH 440/882] Alter "wanted:" to "specified:" in hash mismatch
 output

This makes it even clearer which of the two hashes was specified in the
nix files. Some may think that "wanted" and "got" is obvious, but:
"got" could mean "got in nix file" and "wanted" could mean "want to see in nix file".
---
 src/libexpr/primops/fetchTree.cc      | 2 +-
 src/libstore/build/derivation-goal.cc | 2 +-
 src/libstore/local-store.cc           | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 8d7ae4c14..e6f637a43 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             ? state.store->queryPathInfo(storePath)->narHash
             : hashFile(htSHA256, path);
         if (hash != *expectedHash)
-            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index cc8737fd5..7db83c8be 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -3157,7 +3157,7 @@ void DerivationGoal::registerOutputs()
                        valid. */
                     worker.hashMismatch = true;
                     delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
+                        BuildError("hash mismatch in fixed-output derivation '%s':\n  specified: %s\n     got:    %s",
                             worker.store.printStorePath(drvPath),
                             wanted.to_string(SRI, true),
                             got.to_string(SRI, true)));
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d29a68179..bfad8fb21 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1085,11 +1085,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             auto hashResult = hashSink->finish();
 
             if (hashResult.first != info.narHash)
-                throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
+                throw Error("hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
                     printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
 
             if (hashResult.second != info.narSize)
-                throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
+                throw Error("size mismatch importing path '%s';\n  specified: %s\n  got:       %s",
                     printStorePath(info.path), info.narSize, hashResult.second);
 
             autoGC();

From ff4dea63c9403880500f82ce273713ecf793d2d9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 29 Oct 2020 18:17:39 +0100
Subject: [PATCH 441/882] Generalize extra-* settings

This removes the extra-substituters and extra-sandbox-paths settings
and instead makes every array setting extensible by setting
"extra-<name> = <value>" in the configuration file or passing
"--<name> <value>" on the command line.
---
 .../src/command-ref/conf-file-prefix.md       | 27 ++++++---
 src/libstore/build/derivation-goal.cc         |  6 +-
 src/libstore/daemon.cc                        |  2 -
 src/libstore/globals.cc                       |  9 ++-
 src/libstore/globals.hh                       | 21 +------
 src/libstore/store-api.cc                     |  3 -
 src/libutil/config.cc                         | 58 +++++++++++++++----
 src/libutil/config.hh                         | 11 +++-
 8 files changed, 83 insertions(+), 54 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index e53a3ddf4..3140170ab 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -22,19 +22,30 @@ By default Nix reads settings from the following places:
   - If `NIX_CONFIG` is set, its contents is treated as the contents of
     a configuration file.
 
-The configuration files consist of `name =
-value` pairs, one per line. Other files can be included with a line like
-`include
-path`, where *path* is interpreted relative to the current conf file and
-a missing file is an error unless `!include` is used instead. Comments
+The configuration files consist of `name = value` pairs, one per
+line. Other files can be included with a line like `include path`,
+where *path* is interpreted relative to the current conf file and a
+missing file is an error unless `!include` is used instead. Comments
 start with a `#` character. Here is an example configuration file:
 
     keep-outputs = true       # Nice for developers
     keep-derivations = true   # Idem
 
-You can override settings on the command line using the `--option` flag,
-e.g. `--option keep-outputs
-false`.
+You can override settings on the command line using the `--option`
+flag, e.g. `--option keep-outputs false`. Every configuration setting
+also has a corresponding command line flag, e.g. `--max-jobs 16`; for
+Boolean settings, there are two flags to enable or disable the setting
+(e.g. `--keep-failed` and `--no-keep-failed`).
+
+A configuration setting usually overrides any previous value. However,
+you can prefix the name of the setting by `extra-` to *append* to the
+previous value. For instance,
+
+    substituters = a b
+    extra-substituters = c d
+
+defines the `substituters` setting to be `a b c d`. This is also
+available as a command line flag (e.g. `--extra-substituters`).
 
 The following settings are currently available:
 
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 7db83c8be..3dacb218c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1332,13 +1332,9 @@ void DerivationGoal::startBuilder()
 
         /* Allow a user-configurable set of directories from the
            host file system. */
-        PathSet dirs = settings.sandboxPaths;
-        PathSet dirs2 = settings.extraSandboxPaths;
-        dirs.insert(dirs2.begin(), dirs2.end());
-
         dirsInChroot.clear();
 
-        for (auto i : dirs) {
+        for (auto i : settings.sandboxPaths.get()) {
             if (i.empty()) continue;
             bool optional = false;
             if (i[i.size() - 1] == '?') {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 99d8add92..4dbc7ba38 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -231,8 +231,6 @@ struct ClientSettings
                     settings.set(name, value);
                 else if (setSubstituters(settings.substituters))
                     ;
-                else if (setSubstituters(settings.extraSubstituters))
-                    ;
                 else
                     debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
             } catch (UsageError & e) {
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 4df68d0c9..f38601d6d 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -160,7 +160,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
     {SandboxMode::smDisabled, false},
 });
 
-template<> void BaseSetting<SandboxMode>::set(const std::string & str)
+template<> void BaseSetting<SandboxMode>::set(const std::string & str, bool append)
 {
     if (str == "true") value = smEnabled;
     else if (str == "relaxed") value = smRelaxed;
@@ -168,6 +168,11 @@ template<> void BaseSetting<SandboxMode>::set(const std::string & str)
     else throw UsageError("option '%s' has invalid value '%s'", name, str);
 }
 
+template<> bool BaseSetting<SandboxMode>::isAppendable()
+{
+    return false;
+}
+
 template<> std::string BaseSetting<SandboxMode>::to_string() const
 {
     if (value == smEnabled) return "true";
@@ -198,7 +203,7 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
     });
 }
 
-void MaxBuildJobsSetting::set(const std::string & str)
+void MaxBuildJobsSetting::set(const std::string & str, bool append)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
     else if (!string2Int(str, value))
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 8c63c5b34..eabd83e3f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -25,7 +25,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
         options->addSetting(this);
     }
 
-    void set(const std::string & str) override;
+    void set(const std::string & str, bool append = false) override;
 };
 
 class Settings : public Config {
@@ -413,14 +413,6 @@ public:
     Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
         "Whether to disable sandboxing when the kernel doesn't allow it."};
 
-    Setting<PathSet> extraSandboxPaths{
-        this, {}, "extra-sandbox-paths",
-        R"(
-          A list of additional paths appended to `sandbox-paths`. Useful if
-          you want to extend its default value.
-        )",
-        {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
-
     Setting<size_t> buildRepeat{
         this, 0, "repeat",
         R"(
@@ -599,17 +591,6 @@ public:
         )",
         {"binary-caches"}};
 
-    // FIXME: provide a way to add to option values.
-    Setting<Strings> extraSubstituters{
-        this, {}, "extra-substituters",
-        R"(
-          Additional binary caches appended to those specified in
-          `substituters`. When used by unprivileged users, untrusted
-          substituters (i.e. those not listed in `trusted-substituters`) are
-          silently ignored.
-        )",
-        {"extra-binary-caches"}};
-
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9f21f0434..83d3a1fa1 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1114,9 +1114,6 @@ std::list<ref<Store>> getDefaultSubstituters()
         for (auto uri : settings.substituters.get())
             addStore(uri);
 
-        for (auto uri : settings.extraSubstituters.get())
-            addStore(uri);
-
         stores.sort([](ref<Store> & a, ref<Store> & b) {
             return a->priority < b->priority;
         });
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index eef01bde2..116dd6bfe 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -8,9 +8,18 @@ namespace nix {
 
 bool Config::set(const std::string & name, const std::string & value)
 {
+    bool append = false;
     auto i = _settings.find(name);
-    if (i == _settings.end()) return false;
-    i->second.setting->set(value);
+    if (i == _settings.end()) {
+        if (hasPrefix(name, "extra-")) {
+            i = _settings.find(std::string(name, 6));
+            if (i == _settings.end() || !i->second.setting->isAppendable())
+                return false;
+            append = true;
+        } else
+            return false;
+    }
+    i->second.setting->set(value, append);
     i->second.setting->overriden = true;
     return true;
 }
@@ -180,6 +189,12 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category)
 {
 }
 
+template<typename T>
+bool BaseSetting<T>::isAppendable()
+{
+    return false;
+}
+
 template<typename T>
 void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
@@ -190,9 +205,18 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
         .labels = {"value"},
         .handler = {[=](std::string s) { overriden = true; set(s); }},
     });
+
+    if (isAppendable())
+        args.addFlag({
+            .longName = "extra-" + name,
+            .description = description,
+            .category = category,
+            .labels = {"value"},
+            .handler = {[=](std::string s) { overriden = true; set(s, true); }},
+        });
 }
 
-template<> void BaseSetting<std::string>::set(const std::string & str)
+template<> void BaseSetting<std::string>::set(const std::string & str, bool append)
 {
     value = str;
 }
@@ -203,7 +227,7 @@ template<> std::string BaseSetting<std::string>::to_string() const
 }
 
 template<typename T>
-void BaseSetting<T>::set(const std::string & str)
+void BaseSetting<T>::set(const std::string & str, bool append)
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
     if (!string2Int(str, value))
@@ -217,7 +241,7 @@ std::string BaseSetting<T>::to_string() const
     return std::to_string(value);
 }
 
-template<> void BaseSetting<bool>::set(const std::string & str)
+template<> void BaseSetting<bool>::set(const std::string & str, bool append)
 {
     if (str == "true" || str == "yes" || str == "1")
         value = true;
@@ -248,9 +272,16 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
     });
 }
 
-template<> void BaseSetting<Strings>::set(const std::string & str)
+template<> void BaseSetting<Strings>::set(const std::string & str, bool append)
 {
-    value = tokenizeString<Strings>(str);
+    auto ss = tokenizeString<Strings>(str);
+    if (!append) value.clear();
+    for (auto & s : ss) value.push_back(std::move(s));
+}
+
+template<> bool BaseSetting<Strings>::isAppendable()
+{
+    return true;
 }
 
 template<> std::string BaseSetting<Strings>::to_string() const
@@ -258,7 +289,7 @@ template<> std::string BaseSetting<Strings>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<StringSet>::set(const std::string & str)
+template<> void BaseSetting<StringSet>::set(const std::string & str, bool append)
 {
     value = tokenizeString<StringSet>(str);
 }
@@ -268,9 +299,9 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<StringMap>::set(const std::string & str)
+template<> void BaseSetting<StringMap>::set(const std::string & str, bool append)
 {
-    value.clear();
+    if (!append) value.clear();
     auto kvpairs = tokenizeString<Strings>(str);
     for (auto & s : kvpairs)
     {
@@ -281,6 +312,11 @@ template<> void BaseSetting<StringMap>::set(const std::string & str)
     }
 }
 
+template<> bool BaseSetting<StringMap>::isAppendable()
+{
+    return true;
+}
+
 template<> std::string BaseSetting<StringMap>::to_string() const
 {
     Strings kvstrs;
@@ -301,7 +337,7 @@ template class BaseSetting<Strings>;
 template class BaseSetting<StringSet>;
 template class BaseSetting<StringMap>;
 
-void PathSetting::set(const std::string & str)
+void PathSetting::set(const std::string & str, bool append)
 {
     if (str == "") {
         if (allowEmpty)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 1f5f4e7b9..71e31656d 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -202,7 +202,10 @@ protected:
         assert(created == 123);
     }
 
-    virtual void set(const std::string & value) = 0;
+    virtual void set(const std::string & value, bool append = false) = 0;
+
+    virtual bool isAppendable()
+    { return false; }
 
     virtual std::string to_string() const = 0;
 
@@ -243,7 +246,9 @@ public:
     void operator =(const T & v) { assign(v); }
     virtual void assign(const T & v) { value = v; }
 
-    void set(const std::string & str) override;
+    void set(const std::string & str, bool append = false) override;
+
+    bool isAppendable() override;
 
     virtual void override(const T & v)
     {
@@ -305,7 +310,7 @@ public:
         options->addSetting(this);
     }
 
-    void set(const std::string & str) override;
+    void set(const std::string & str, bool append = false) override;
 
     Path operator +(const char * p) const { return value + p; }
 

From 7f56cf67bac3731ed8e217170eb548bf0fd2cfcb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 29 Oct 2020 18:26:35 +0100
Subject: [PATCH 442/882] Fix assertion failure in tab completion for --option

---
 src/libmain/common-args.cc | 2 +-
 src/libutil/config.cc      | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 9151a0344..3e4e475e5 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
                 globalConfig.getSettings(settings);
                 for (auto & s : settings)
                     if (hasPrefix(s.first, prefix))
-                        completions->add(s.first, s.second.description);
+                        completions->add(s.first, fmt("Set the `%s` setting.", s.first));
             }
         }
     });
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 116dd6bfe..be957dfe3 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -200,7 +200,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
     args.addFlag({
         .longName = name,
-        .description = description,
+        .description = fmt("Set the `%s` setting.", name),
         .category = category,
         .labels = {"value"},
         .handler = {[=](std::string s) { overriden = true; set(s); }},
@@ -209,7 +209,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
     if (isAppendable())
         args.addFlag({
             .longName = "extra-" + name,
-            .description = description,
+            .description = fmt("Append to the `%s` setting.", name),
             .category = category,
             .labels = {"value"},
             .handler = {[=](std::string s) { overriden = true; set(s, true); }},
@@ -260,13 +260,13 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
 {
     args.addFlag({
         .longName = name,
-        .description = description,
+        .description = fmt("Enable the `%s` setting.", name),
         .category = category,
         .handler = {[=]() { override(true); }}
     });
     args.addFlag({
         .longName = "no-" + name,
-        .description = description,
+        .description = fmt("Disable the `%s` setting.", name),
         .category = category,
         .handler = {[=]() { override(false); }}
     });

From b809c48ebba7b628a5fb0a0e284cf7068589d479 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 30 Oct 2020 11:01:33 +0100
Subject: [PATCH 443/882] nix-shell.md: evaluated -> run

Use "run" to avoid confusion with Nix evaluation.
"evaluated" was intended to reference bash eval but it's ambiguous.
---
 doc/manual/src/command-ref/nix-shell.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index d1266930e..8c77923d0 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -32,7 +32,7 @@ URL of a tarball that will be downloaded and unpacked to a temporary
 location. The tarball must include a single top-level directory
 containing at least a file named `default.nix`.
 
-If the derivation defines the variable `shellHook`, it will be evaluated
+If the derivation defines the variable `shellHook`, it will be run
 after `$stdenv/setup` has been sourced. Since this hook is not executed
 by regular Nix builds, it allows you to perform initialisation specific
 to `nix-shell`. For example, the derivation attribute

From d4c5d8d32a4bc5b3eaa63add23dca9d755dc5a74 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 30 Oct 2020 11:12:28 +0100
Subject: [PATCH 444/882] nix-shell.md: Extend shellHook example

---
 doc/manual/src/command-ref/nix-shell.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 8c77923d0..88b675e71 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -41,10 +41,12 @@ to `nix-shell`. For example, the derivation attribute
 shellHook =
   ''
     echo "Hello shell"
+    export SOME_API_TOKEN="$(cat ~/.config/some-app/api-token)"
   '';
 ```
 
-will cause `nix-shell` to print `Hello shell`.
+will cause `nix-shell` to print `Hello shell` and set the `SOME_API_TOKEN`
+environment variable to a user-configured value.
 
 # Options
 

From dc5696b84f55a6706cddc3d747ef1aeffb564f43 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 30 Oct 2020 12:00:43 +0100
Subject: [PATCH 445/882] Fix test

---
 src/libutil/tests/config.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index c7777a21f..c305af9f5 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -80,8 +80,8 @@ namespace nix {
         class TestSetting : public AbstractSetting {
             public:
             TestSetting() : AbstractSetting("test", "test", {}) {}
-            void set(const std::string & value) {}
-            std::string to_string() const { return {}; }
+            void set(const std::string & value, bool append) override {}
+            std::string to_string() const override { return {}; }
         };
 
         Config config;

From c4d903ddb009aa6472530699e154d85a24eac51d Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 30 Oct 2020 20:55:53 +0100
Subject: [PATCH 446/882] Fix memory corruption caused by GC-invisible
 coroutine stacks

Crucially this introduces BoehmGCStackAllocator, but it also
adds a bunch of wiring to avoid making libutil depend on bdw-gc.

Part of the solutions for #4178, #4200
---
 src/libexpr/eval.cc      | 26 ++++++++++++++++++++++++++
 src/libexpr/local.mk     |  2 +-
 src/libutil/serialise.cc | 35 ++++++++++++++++++++++++++++++++++-
 src/libutil/serialise.hh | 14 ++++++++++++++
 4 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 4de87d647..ea7ba0a6a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -27,6 +27,10 @@
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
+#include <boost/coroutine2/coroutine.hpp>
+#include <boost/coroutine2/protected_fixedsize_stack.hpp>
+#include <boost/context/stack_context.hpp>
+
 #endif
 
 namespace nix {
@@ -220,6 +224,26 @@ static void * oomHandler(size_t requested)
     /* Convert this to a proper C++ exception. */
     throw std::bad_alloc();
 }
+
+class BoehmGCStackAllocator : public StackAllocator {
+  boost::coroutines2::protected_fixedsize_stack stack;
+
+  public:
+    boost::context::stack_context allocate() override {
+        auto sctx = stack.allocate();
+        GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+        return sctx;
+    }
+
+    void deallocate(boost::context::stack_context sctx) override {
+        GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+        stack.deallocate(sctx);
+    }
+
+};
+
+static BoehmGCStackAllocator boehmGCStackAllocator;
+
 #endif
 
 
@@ -257,6 +281,8 @@ void initGC()
 
     GC_set_oom_fn(oomHandler);
 
+    StackAllocator::defaultAllocator = &boehmGCStackAllocator;
+
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
        we don't need to garbage collect at all.  (Collection has a
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 687a8ccda..a5422169d 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS =
+libexpr_LDFLAGS = -lboost_context
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 5c9f6f901..28f6968d0 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -171,6 +171,39 @@ size_t StringSource::read(unsigned char * data, size_t len)
 #error Coroutines are broken in this version of Boost!
 #endif
 
+/* A concrete datatype allow virtual dispatch of stack allocation methods. */
+struct VirtualStackAllocator {
+    StackAllocator *allocator = StackAllocator::defaultAllocator;
+
+    boost::context::stack_context allocate() {
+        return allocator->allocate();
+    }
+
+    void deallocate(boost::context::stack_context sctx) {
+        allocator->deallocate(sctx);
+    }
+};
+
+
+/* This class reifies the default boost coroutine stack allocation strategy with
+   a virtual interface. */
+class DefaultStackAllocator : public StackAllocator {
+    boost::coroutines2::default_stack stack;
+
+    boost::context::stack_context allocate() {
+        return stack.allocate();
+    }
+
+    void deallocate(boost::context::stack_context sctx) {
+        deallocate(sctx);
+    }
+};
+
+static DefaultStackAllocator defaultAllocatorSingleton;
+
+StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
+
+
 std::unique_ptr<Source> sinkToSource(
     std::function<void(Sink &)> fun,
     std::function<void()> eof)
@@ -195,7 +228,7 @@ std::unique_ptr<Source> sinkToSource(
         size_t read(unsigned char * data, size_t len) override
         {
             if (!coro)
-                coro = coro_t::pull_type([&](coro_t::push_type & yield) {
+                coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
                     LambdaSink sink([&](const unsigned char * data, size_t len) {
                             if (len) yield(std::string((const char *) data, len));
                         });
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index d7fe0b81e..5c7d3ce76 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -5,6 +5,7 @@
 #include "types.hh"
 #include "util.hh"
 
+namespace boost::context { struct stack_context; }
 
 namespace nix {
 
@@ -497,5 +498,18 @@ struct FramedSink : nix::BufferedSink
     };
 };
 
+/* Stack allocation strategy for sinkToSource.
+   Mutable to avoid a boehm gc dependency in libutil.
+
+   boost::context doesn't provide a virtual class, so we define our own.
+ */
+struct StackAllocator {
+    virtual boost::context::stack_context allocate() = 0;
+    virtual void deallocate(boost::context::stack_context sctx) = 0;
+
+    /* The stack allocator to use in sinkToSource and potentially elsewhere.
+       It is reassigned by the initGC() method in libexpr. */
+    static StackAllocator *defaultAllocator;
+};
 
 }

From 2192cac634dcd13c3365f4dfeb5f393f4fd9e327 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 30 Oct 2020 21:47:34 +0100
Subject: [PATCH 447/882] Fix RemoteStore pool deadlock in filterSource etc

---
 src/libstore/remote-store.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 488270f48..ccbd3bb1d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -471,9 +471,14 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         worker_proto::write(*this, conn->to, references);
         conn->to << repair;
 
-        conn.withFramedSink([&](Sink & sink) {
-            dump.drainInto(sink);
-        });
+        // The dump source may invoke the store, so we need to make some room.
+        connections->incCapacity();
+        {
+            Finally cleanup([&]() { connections->decCapacity(); });
+            conn.withFramedSink([&](Sink & sink) {
+                dump.drainInto(sink);
+            });
+        }
 
         auto path = parseStorePath(readString(conn->from));
         return readValidPathInfo(conn, path);

From b43c13a9161daf1801188e61104debafa5243fe1 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 30 Oct 2020 23:18:24 +0100
Subject: [PATCH 448/882] BoehmGCStackAllocator: increase stack size to 8MB

The default stack size was not based on the normal stack size and
was too small.
---
 src/libexpr/eval.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index ea7ba0a6a..486a9fc1a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -226,7 +226,12 @@ static void * oomHandler(size_t requested)
 }
 
 class BoehmGCStackAllocator : public StackAllocator {
-  boost::coroutines2::protected_fixedsize_stack stack;
+  boost::coroutines2::protected_fixedsize_stack stack {
+    // We allocate 8 MB, the default max stack size on NixOS.
+    // A smaller stack might be quicker to allocate but reduces the stack
+    // depth available for source filter expressions etc.
+    std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
+    };
 
   public:
     boost::context::stack_context allocate() override {

From e8a45d07bccc315de88b8434393082eee6e944a8 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 31 Oct 2020 23:52:09 +0100
Subject: [PATCH 449/882] Restore RestrictedStore.addToStoreFromDump
 implementation

It was accidentally removed in commit ca30abb3fb36440e5a13161c39647189036fc18f
---
 src/libstore/build/derivation-goal.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3dacb218c..19d96dd8f 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -2074,6 +2074,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
         return path;
     }
 
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+    {
+        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
+        goal.addDependency(path);
+        return path;
+    }
+
     void narFromPath(const StorePath & path, Sink & sink) override
     {
         if (!goal.isAllowed(path))

From db5424bf09886afc1c81db36766522f68fc66ba8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 2 Nov 2020 13:57:58 +0100
Subject: [PATCH 450/882] Don't send eval-related settings to the daemon

---
 src/libstore/remote-store.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 488270f48..fb52ca6d0 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -11,6 +11,7 @@
 #include "finally.hh"
 #include "logging.hh"
 #include "callback.hh"
+#include "filetransfer.hh"
 
 namespace nix {
 
@@ -171,7 +172,8 @@ void RemoteStore::setOptions(Connection & conn)
 
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
         std::map<std::string, Config::SettingInfo> overrides;
-        globalConfig.getSettings(overrides, true);
+        settings.getSettings(overrides, true); // libstore settings
+        fileTransferSettings.getSettings(overrides, true);
         overrides.erase(settings.keepFailed.name);
         overrides.erase(settings.keepGoing.name);
         overrides.erase(settings.tryFallback.name);

From 8b15650e7421cf9433f04c32ac73601aada1e3ab Mon Sep 17 00:00:00 2001
From: mkenigs <40775676+mkenigs@users.noreply.github.com>
Date: Mon, 2 Nov 2020 09:32:05 -0600
Subject: [PATCH 451/882] docs: consistent console prompt (#4213)

Everywhere else a $ is used
---
 doc/manual/src/installation/installing-binary.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index 8b8d1d738..ae7fd458b 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -195,7 +195,7 @@ If you are comfortable navigating these tradeoffs, you can encrypt the
 volume with something along the lines of:
 
 ```console
-alice$ diskutil apfs enableFileVault /nix -user disk
+$ diskutil apfs enableFileVault /nix -user disk
 ```
 
 ## Symlink the Nix store to a custom location
@@ -234,13 +234,13 @@ as a helpful reference if you run into trouble.
     without a reboot:
     
     ```console
-    alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+    $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
     ```
 
 3.  Create the new APFS volume with diskutil:
     
     ```console
-    alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
+    $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
     ```
 
 4.  Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't
@@ -280,10 +280,10 @@ it somewhere (e.g. in `/tmp`), and then run the script named `install`
 inside the binary tarball:
 
 ```console
-alice$ cd /tmp
-alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-alice$ cd nix-1.8-x86_64-darwin
-alice$ ./install
+$ cd /tmp
+$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
+$ cd nix-1.8-x86_64-darwin
+$ ./install
 ```
 
 If you need to edit the multi-user installation script to use different

From 7cf874c17d466d5cffdb0eb6215fcfe8930ed757 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 2 Nov 2020 18:46:44 +0100
Subject: [PATCH 452/882] Don't use readDerivation() in addValidPath()

readDerivation() requires a valid path.

Fixes #4210.
---
 src/libstore/local-store.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index bfad8fb21..2892b0407 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -623,7 +623,10 @@ uint64_t LocalStore::addValidPath(State & state,
        efficiently query whether a path is an output of some
        derivation. */
     if (info.path.isDerivation()) {
-        auto drv = readDerivation(info.path);
+        auto drv = parseDerivation(
+            *this,
+            readFile(Store::toRealPath(info.path)),
+            Derivation::nameFromPath(info.path));
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -1000,7 +1003,11 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         for (auto & i : infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                checkDerivationOutputs(i.path, readDerivation(i.path));
+                checkDerivationOutputs(i.path,
+                    parseDerivation(
+                        *this,
+                        readFile(Store::toRealPath(i.path)),
+                        Derivation::nameFromPath(i.path)));
             }
 
         /* Do a topological sort of the paths.  This will throw an

From 550e11f077ae508abde5a33998a9d4029880e7b2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 2 Nov 2020 19:07:37 +0100
Subject: [PATCH 453/882] nix repl: Fix handling of multi-line expressions

---
 src/nix/repl.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 9ff386b1d..71794a309 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -212,7 +212,7 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
         try {
             if (!removeWhitespace(input).empty() && !processLine(input)) return;
         } catch (ParseError & e) {
-            if (e.msg().find("unexpected $end") != std::string::npos) {
+            if (e.msg().find("unexpected end of file") != std::string::npos) {
                 // For parse errors on incomplete input, we continue waiting for the next line of
                 // input without clearing the input so far.
                 continue;
@@ -220,9 +220,9 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
               printMsg(lvlError, e.msg());
             }
         } catch (Error & e) {
-          printMsg(lvlError, e.msg());
+            printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
-          printMsg(lvlError, e.msg());
+            printMsg(lvlError, e.msg());
         }
 
         // We handled the current input fully, so we should clear it

From 8cd2ff69c38913d5fcddeb0a012ba0cf34de0686 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 2 Nov 2020 15:50:14 -0500
Subject: [PATCH 454/882] nix-copy-closure: verify it works with drvs

Creates test coverage for #4210 and 7cf874c17d466d5cffdb0eb6215fcfe8930ed757
---
 tests/nix-copy-closure.nix | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index e5f6a0f12..1b63a3fca 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -7,14 +7,14 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") {
   extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
 };
 
-makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
+makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in {
   name = "nix-copy-closure";
 
   nodes =
     { client =
         { config, lib, pkgs, ... }:
         { virtualisation.writableStore = true;
-          virtualisation.pathsInNixDB = [ pkgA ];
+          virtualisation.pathsInNixDB = [ pkgA pkgD.drvPath ];
           nix.binaryCaches = lib.mkForce [ ];
         };
 
@@ -60,6 +60,12 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
 
     # Copy the closure of package C via the SSH substituter.
     client.fail("nix-store -r ${pkgC}")
+
+    # Copy the derivation of package D's derivation from the client to the server.
+    server.fail("nix-store --check-validity ${pkgD.drvPath}")
+    client.succeed("nix-copy-closure --to server --gzip ${pkgD.drvPath} >&2")
+    server.succeed("nix-store --check-validity ${pkgD.drvPath}")
+
     # FIXME
     # client.succeed(
     #   "nix-store --option use-ssh-substituter true"

From 797a52e31d97381e0be7e0a80e43f8cb259f8a6b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Nov 2020 12:26:53 +0100
Subject: [PATCH 455/882] Add FIXME

---
 src/libstore/local-store.hh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index f1e2ab7f9..5e07261a6 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -90,6 +90,8 @@ private:
         std::unique_ptr<PublicKeys> publicKeys;
     };
 
+    // FIXME: get rid of recursive_mutex, it hides recursive SQLite
+    // queries.
     Sync<State, std::recursive_mutex> _state;
 
 public:

From e8c379555fa0441c5ab83b8e5a3a106d69fe2507 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Nov 2020 12:52:57 +0100
Subject: [PATCH 456/882] LocalStore: Get rid of recursive_mutex

---
 src/libstore/build/derivation-goal.cc |  6 +++---
 src/libstore/local-store.cc           | 23 ++++++++++++++++-------
 src/libstore/local-store.hh           |  4 +---
 src/libstore/path-info.hh             |  2 +-
 src/nix-store/nix-store.cc            |  2 +-
 5 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 19d96dd8f..bf2bad62c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -3245,7 +3245,7 @@ void DerivationGoal::registerOutputs()
             if (!oldInfo.ultimate) {
                 oldInfo.ultimate = true;
                 worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({ std::move(oldInfo) });
+                worker.store.registerValidPaths({{oldInfo.path, oldInfo}});
             }
 
             continue;
@@ -3275,7 +3275,7 @@ void DerivationGoal::registerOutputs()
            isn't statically known so that we can safely unlock the path before
            the next iteration */
         if (newInfo.ca)
-            worker.store.registerValidPaths({newInfo});
+            worker.store.registerValidPaths({{newInfo.path, newInfo}});
 
         infos.emplace(outputName, std::move(newInfo));
     }
@@ -3350,7 +3350,7 @@ void DerivationGoal::registerOutputs()
     {
         ValidPathInfos infos2;
         for (auto & [outputName, newInfo] : infos) {
-            infos2.push_back(newInfo);
+            infos2.insert_or_assign(newInfo.path, newInfo);
         }
         worker.store.registerValidPaths(infos2);
     }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2892b0407..e4e404dca 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -7,6 +7,7 @@
 #include "nar-info.hh"
 #include "references.hh"
 #include "callback.hh"
+#include "topo-sort.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -962,9 +963,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
 
 void LocalStore::registerValidPath(const ValidPathInfo & info)
 {
-    ValidPathInfos infos;
-    infos.push_back(info);
-    registerValidPaths(infos);
+    registerValidPaths({{info.path, info}});
 }
 
 
@@ -982,7 +981,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         SQLiteTxn txn(state->db);
         StorePathSet paths;
 
-        for (auto & i : infos) {
+        for (auto & [_, i] : infos) {
             assert(i.narHash.type == htSHA256);
             if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
@@ -991,7 +990,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
             paths.insert(i.path);
         }
 
-        for (auto & i : infos) {
+        for (auto & [_, i] : infos) {
             auto referrer = queryValidPathId(*state, i.path);
             for (auto & j : i.references)
                 state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
@@ -1000,7 +999,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         /* Check that the derivation outputs are correct.  We can't do
            this in addValidPath() above, because the references might
            not be valid yet. */
-        for (auto & i : infos)
+        for (auto & [_, i] : infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
                 checkDerivationOutputs(i.path,
@@ -1014,7 +1013,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
            error if a cycle is detected and roll back the
            transaction.  Cycles can only occur when a derivation
            has multiple outputs. */
-        topoSortPaths(paths);
+        topoSort(paths,
+            {[&](const StorePath & path) {
+                auto i = infos.find(path);
+                return i == infos.end() ? StorePathSet() : i->second.references;
+            }},
+            {[&](const StorePath & path, const StorePath & parent) {
+                return BuildError(
+                    "cycle detected in the references of '%s' from '%s'",
+                    printStorePath(path),
+                    printStorePath(parent));
+            }});
 
         txn.commit();
     });
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 5e07261a6..d4435220d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -90,9 +90,7 @@ private:
         std::unique_ptr<PublicKeys> publicKeys;
     };
 
-    // FIXME: get rid of recursive_mutex, it hides recursive SQLite
-    // queries.
-    Sync<State, std::recursive_mutex> _state;
+    Sync<State> _state;
 
 public:
 
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 8ff5c466e..de87f8b33 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -107,6 +107,6 @@ struct ValidPathInfo
     virtual ~ValidPathInfo() { }
 };
 
-typedef list<ValidPathInfo> ValidPathInfos;
+typedef std::map<StorePath, ValidPathInfo> ValidPathInfos;
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 14baabc36..6c2702bfe 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -516,7 +516,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
                 info->narHash = hash.first;
                 info->narSize = hash.second;
             }
-            infos.push_back(std::move(*info));
+            infos.insert_or_assign(info->path, *info);
         }
     }
 

From 5e6eabe1551f3fa2a463fced39a4a48b504ed1ab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Nov 2020 14:44:52 +0100
Subject: [PATCH 457/882] Fix error message 'assertion failed at'

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 4de87d647..cb8e0a0de 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1405,7 +1405,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
+        throwAssertionError(pos, "assertion '%1%' failed", out.str());
     }
     body->eval(state, env, v);
 }

From 3a63fc6cd515fb009b17d864fede23a356832a5e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Wed, 21 Oct 2020 21:31:19 +0200
Subject: [PATCH 458/882] Allow substituting paths when building remotely using
 `ssh-ng://`

Until now, it was not possible to substitute missing paths from e.g.
`https://cache.nixos.org` on a remote server when building on it using
the new `ssh-ng` protocol.

This is because every store implementation except legacy `ssh://`
ignores the substitution flag passed to `Store::queryValidPaths` while
the `legacy-ssh-store` substitutes the remote store using
`cmdQueryValidPaths` when the remote store is opened with `nix-store
--serve`.

This patch slightly modifies the daemon protocol to allow passing an
integer value suggesting whether to substitute missing paths during
`wopQueryValidPaths`. To implement this on the daemon-side, the
substitution logic from `nix-store --serve` has been moved into a
protected method named `Store::substitutePaths` which gets currently
called from `LocalStore::queryValidPaths` and `Store::queryValidPaths`
if `maybeSubstitute` is `true`.

Fixes #2770
---
 flake.nix                       |  3 ++-
 src/libstore/daemon.cc          | 11 ++++++++++-
 src/libstore/remote-store.cc    |  3 +++
 src/libstore/store-api.cc       | 22 ++++++++++++++++++++++
 src/libstore/store-api.hh       |  5 +++++
 src/libstore/worker-protocol.hh |  2 +-
 src/nix-store/nix-store.cc      | 23 +----------------------
 7 files changed, 44 insertions(+), 25 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2abbdff53..8ff048be3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -12,7 +12,7 @@
       versionSuffix =
         if officialRelease
         then ""
-        else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
+        else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
       officialRelease = false;
 
@@ -117,6 +117,7 @@
 
         nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
           name = "nix-${version}";
+          inherit version;
 
           src = self;
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 4dbc7ba38..60cca4fda 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopQueryValidPaths: {
         auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+
+        SubstituteFlag substitute = NoSubstitute;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
+            substitute = readInt(from) ? Substitute : NoSubstitute;
+        }
+
         logger->startWork();
-        auto res = store->queryValidPaths(paths);
+        if (substitute) {
+            store->substitutePaths(paths);
+        }
+        auto res = store->queryValidPaths(paths, substitute);
         logger->stopWork();
         worker_proto::write(*store, to, res);
         break;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b6f70057d..c27a0f278 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
     } else {
         conn->to << wopQueryValidPaths;
         worker_proto::write(*this, conn->to, paths);
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
+            conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
+        }
         conn.processStderr();
         return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 83d3a1fa1..3129d6637 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath,
 }
 
 
+void Store::substitutePaths(const StorePathSet & paths)
+{
+    std::vector<StorePathWithOutputs> paths2;
+    for (auto & path : paths)
+        if (!path.isDerivation())
+            paths2.push_back({path});
+    uint64_t downloadSize, narSize;
+    StorePathSet willBuild, willSubstitute, unknown;
+    queryMissing(paths2,
+        willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+    if (!willSubstitute.empty())
+        try {
+            std::vector<StorePathWithOutputs> subs;
+            for (auto & p : willSubstitute) subs.push_back({p});
+            buildPaths(subs);
+        } catch (Error & e) {
+            logWarning(e.info());
+        }
+}
+
+
 StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
 {
     struct State
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f77bc21d1..9a373b561 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -360,6 +360,11 @@ protected:
 
 public:
 
+    /* If requested, substitute missing paths. This
+       implements nix-copy-closure's --use-substitutes
+       flag. */
+    void substitutePaths(const StorePathSet & paths);
+
     /* Query which of the given paths is valid. Optionally, try to
        substitute missing paths. */
     virtual StorePathSet queryValidPaths(const StorePathSet & paths,
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b3705578e..63bd6ea49 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x11a
+#define PROTOCOL_VERSION 0x11b
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 6c2702bfe..54394e921 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -830,29 +830,8 @@ static void opServe(Strings opFlags, Strings opArgs)
                     for (auto & path : paths)
                         store->addTempRoot(path);
 
-                /* If requested, substitute missing paths. This
-                   implements nix-copy-closure's --use-substitutes
-                   flag. */
                 if (substitute && writeAllowed) {
-                    /* Filter out .drv files (we don't want to build anything). */
-                    std::vector<StorePathWithOutputs> paths2;
-                    for (auto & path : paths)
-                        if (!path.isDerivation())
-                            paths2.push_back({path});
-                    uint64_t downloadSize, narSize;
-                    StorePathSet willBuild, willSubstitute, unknown;
-                    store->queryMissing(paths2,
-                        willBuild, willSubstitute, unknown, downloadSize, narSize);
-                    /* FIXME: should use ensurePath(), but it only
-                       does one path at a time. */
-                    if (!willSubstitute.empty())
-                        try {
-                            std::vector<StorePathWithOutputs> subs;
-                            for (auto & p : willSubstitute) subs.push_back({p});
-                            store->buildPaths(subs);
-                        } catch (Error & e) {
-                            logWarning(e.info());
-                        }
+                    store->substitutePaths(paths);
                 }
 
                 worker_proto::write(*store, out, store->queryValidPaths(paths));

From b87f84cf55b4ca666b31c511e2489789e3322da4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Nov 2020 15:04:34 +0100
Subject: [PATCH 459/882] Fix appending to Setting<StringSet>

Fixes: warning: unknown setting 'extra-sandbox-paths'
---
 src/libutil/config.cc | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index be957dfe3..7af3e7883 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -291,7 +291,14 @@ template<> std::string BaseSetting<Strings>::to_string() const
 
 template<> void BaseSetting<StringSet>::set(const std::string & str, bool append)
 {
-    value = tokenizeString<StringSet>(str);
+    if (!append) value.clear();
+    for (auto & s : tokenizeString<StringSet>(str))
+        value.insert(s);
+}
+
+template<> bool BaseSetting<StringSet>::isAppendable()
+{
+    return true;
 }
 
 template<> std::string BaseSetting<StringSet>::to_string() const
@@ -302,9 +309,7 @@ template<> std::string BaseSetting<StringSet>::to_string() const
 template<> void BaseSetting<StringMap>::set(const std::string & str, bool append)
 {
     if (!append) value.clear();
-    auto kvpairs = tokenizeString<Strings>(str);
-    for (auto & s : kvpairs)
-    {
+    for (auto & s : tokenizeString<Strings>(str)) {
         auto eq = s.find_first_of('=');
         if (std::string::npos != eq)
             value.emplace(std::string(s, 0, eq), std::string(s, eq + 1));

From fb7735e4cf3a1ee6337bf1f2ee15204bb11304b2 Mon Sep 17 00:00:00 2001
From: Sebastian Ullrich <sebasti@nullri.ch>
Date: Sat, 7 Nov 2020 15:00:22 +0100
Subject: [PATCH 460/882] nix develop: Preserve stdin with `-c`

---
 src/nix/develop.cc | 4 ++--
 tests/nix-shell.sh | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 8fea7ee9c..457d94382 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -474,9 +474,9 @@ struct CmdDevelop : Common, MixEnvironment
             ignoreException();
         }
 
-        // If running a phase, don't want an interactive shell running after
+        // If running a phase or single command, don't want an interactive shell running after
         // Ctrl-C, so don't pass --rcfile
-        auto args = phase ? Strings{std::string(baseNameOf(shell)), rcFilePath}
+        auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath}
             : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
 
         restoreAffinity();
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 1228bb04f..dfe8ed0c1 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -58,6 +58,7 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test 'nix develop'.
 nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+echo foo | nix develop -f shell.nix shellDrv -c cat  # preserve stdin with `-c`
 
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)

From 579b953231599f111dcc9fac95491458c9f6ef08 Mon Sep 17 00:00:00 2001
From: Sebastian Ullrich <sebasti@nullri.ch>
Date: Mon, 9 Nov 2020 17:50:51 +0100
Subject: [PATCH 461/882] Make test case more precise
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
---
 tests/nix-shell.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index dfe8ed0c1..3b000a049 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -58,7 +58,8 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test 'nix develop'.
 nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
-echo foo | nix develop -f shell.nix shellDrv -c cat  # preserve stdin with `-c`
+# Preserve stdin with `-c`
+echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)

From 3f24a417dacd5a5964f781f4d704a5253401d380 Mon Sep 17 00:00:00 2001
From: Sebastian Ullrich <sebasti@nullri.ch>
Date: Mon, 9 Nov 2020 17:57:39 +0100
Subject: [PATCH 462/882] Add test case for incidentally fixed #4228

---
 tests/nix-shell.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 3b000a049..7b2be650a 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -58,9 +58,13 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test 'nix develop'.
 nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
-# Preserve stdin with `-c`
+
+# Ensure `nix develop -c` preserves stdin
 echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 
+# Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
+nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo
+
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)
 [[ -n $stdenv ]]

From 0ed7c957bed18d963df65d55f0c0ffc79dee656d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Nov 2020 23:21:55 +0000
Subject: [PATCH 463/882] Bump cachix/install-nix-action from v11 to v12
 (#4237)

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v11 to v12.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v11...07da2520eebede906fbeefa9dd0a2b635323909d)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 829111b67..021642f4c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,6 +12,6 @@ jobs:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v11
+    - uses: cachix/install-nix-action@v12
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

From 107c91f5fe6248548c292d28d0ad53c0de7ceeba Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 9 Nov 2020 16:48:35 -0700
Subject: [PATCH 464/882] auto-call error

---
 src/libexpr/eval.cc | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d6366050c..e52e8dcf2 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1370,7 +1370,28 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             if (j != args.end()) {
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
-                throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+                throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%')
+  An 'auto-call' is when a nix expression is evaluated without any external arguments.  If that
+  nix expression is a function, and that function's arguments all have default values, then all is well.
+
+  But if the function arguments don't have default values, then evaluation fails.
+
+  The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage.
+    # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.
+    # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function.
+    { stdenv }:
+    stdenv.mkDerivation  {
+    ...
+
+    # in 'auto-call' format: nixpkgs is imported explicitly, and used directly.
+    let
+      nixpkgs = import <nixpkgs> {};
+    in
+      nixpkgs.stdenv.mkDerivation {
+    ...
+
+  See this nix pill for more information re callPackage format:
+  https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name);
             }
         }
     }

From 6c2933a8d72f9328a2931a8166439bed96b80f24 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 9 Nov 2020 17:04:52 -0700
Subject: [PATCH 465/882] add position

---
 src/libexpr/eval.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e52e8dcf2..11f2e31ce 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1370,11 +1370,11 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             if (j != args.end()) {
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
-                throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%')
+                throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%')
   An 'auto-call' is when a nix expression is evaluated without any external arguments.  If that
   nix expression is a function, and that function's arguments all have default values, then all is well.
 
-  But if the function arguments don't have default values, then evaluation fails.
+  But if the function arguments don't have default values, evaluation fails.
 
   The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage.
     # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.

From 9f2b25ce55e38fc5772fcb65bd98651255f6a49b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 9 Nov 2020 17:17:47 -0700
Subject: [PATCH 466/882] remove unused ftn; reformat line breaks

---
 src/libexpr/eval.cc | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 11f2e31ce..0446376f3 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -636,11 +636,6 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
     });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
-{
-    throw TypeError(s, s1);
-}
-
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
     throw TypeError({
@@ -1371,12 +1366,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
                 throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%')
-  An 'auto-call' is when a nix expression is evaluated without any external arguments.  If that
-  nix expression is a function, and that function's arguments all have default values, then all is well.
+  An 'auto-call' is when a nix expression is evaluated without any external arguments.
+  If that nix expression is a function, and that function's arguments all have default
+  values, then all is well.
 
   But if the function arguments don't have default values, evaluation fails.
 
-  The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage.
+  The classic case for this error is evaluating a nix file with nix-build that expects
+  to be evaluated by callPackage.
     # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.
     # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function.
     { stdenv }:

From d8ef423a189b95f9f6e0a385a0f4474bebf297ef Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 9 Nov 2020 19:16:50 -0700
Subject: [PATCH 467/882] error message formatting

---
 src/libexpr/eval.cc | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0446376f3..2b697882b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1366,29 +1366,30 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
                 throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%')
-  An 'auto-call' is when a nix expression is evaluated without any external arguments.
-  If that nix expression is a function, and that function's arguments all have default
-  values, then all is well.
 
-  But if the function arguments don't have default values, evaluation fails.
+An 'auto-call' is when a nix expression is evaluated without any external arguments.
+If that nix expression is a function, and that function's arguments all have default
+values, then all is well.
 
-  The classic case for this error is evaluating a nix file with nix-build that expects
-  to be evaluated by callPackage.
-    # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.
-    # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function.
-    { stdenv }:
-    stdenv.mkDerivation  {
-    ...
+But if the function arguments don't have default values, evaluation fails.
 
-    # in 'auto-call' format: nixpkgs is imported explicitly, and used directly.
-    let
-      nixpkgs = import <nixpkgs> {};
-    in
-      nixpkgs.stdenv.mkDerivation {
-    ...
+The classic case for this error is evaluating a nix file with nix-build that expects
+to be evaluated by callPackage.
+  # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.
+  # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function.
+  { stdenv }:
+  stdenv.mkDerivation  {
+  ...
 
-  See this nix pill for more information re callPackage format:
-  https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name);
+  # in 'auto-call' format: nixpkgs is imported explicitly, and used directly.
+  let
+    nixpkgs = import <nixpkgs> {};
+  in
+    nixpkgs.stdenv.mkDerivation {
+  ...
+
+More about callPackage: 
+https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name);
             }
         }
     }

From 108a2dab7e460533064b24f5dff05adc453acb27 Mon Sep 17 00:00:00 2001
From: "Ricardo M. Correia" <rcorreia@wizy.org>
Date: Tue, 10 Nov 2020 04:24:55 +0100
Subject: [PATCH 468/882] Fix stack overflow introduced in #4206

---
 src/libutil/serialise.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 28f6968d0..038ede049 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -195,7 +195,7 @@ class DefaultStackAllocator : public StackAllocator {
     }
 
     void deallocate(boost::context::stack_context sctx) {
-        deallocate(sctx);
+        stack.deallocate(sctx);
     }
 };
 

From 4864df6d6b7f5fc5c0815bf7733c40663ea856d7 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Tue, 10 Nov 2020 08:48:49 -0600
Subject: [PATCH 469/882] enable Darwin.arm64 to install x86_64 binary

Throwing @thefloweringash under the bus if this doesn't work, but it
sounds like Apple Silicon devices can use the x86_64 binary for now.

Fixes #4058
---
 scripts/install.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/install.in b/scripts/install.in
index 39fae37e3..9a281d776 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -29,6 +29,8 @@ case "$(uname -s).$(uname -m)" in
     Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
     Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
     Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
+    # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;;
+    Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 

From 4badb6943f73e3d740d4de793eb9860e26842bc1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Nov 2020 23:22:45 +0100
Subject: [PATCH 470/882] Fix use of dirty Git/Mercurial inputs with chroot
 stores

Fixes:

  $ nix build --store /tmp/nix /home/eelco/Dev/patchelf#hydraJobs.build.x86_64-linux
  warning: Git tree '/home/eelco/Dev/patchelf' is dirty
  error: --- RestrictedPathError ------------------------------------------------------------------------------------------- nix
  access to path '/tmp/nix/nix/store/xmkvfmffk7xfnazykb5kx999aika8an4-source/flake.nix' is forbidden in restricted mode
  (use '--show-trace' to show detailed location information)
---
 src/libfetchers/git.cc       | 2 +-
 src/libfetchers/mercurial.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a6411b02b..e7712c5fd 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -273,7 +273,7 @@ struct GitInputScheme : InputScheme
                     haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
 
                 return {
-                    Tree(store->printStorePath(storePath), std::move(storePath)),
+                    Tree(store->toRealPath(storePath), std::move(storePath)),
                     input
                 };
             }
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 7d3d52751..41f60c45c 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -166,7 +166,7 @@ struct MercurialInputScheme : InputScheme
                 auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {
-                    Tree(store->printStorePath(storePath), std::move(storePath)),
+                    Tree(store->toRealPath(storePath), std::move(storePath)),
                     input
                 };
             }

From 7d9037035ef8bfe0b7ae00a9e3b139ae83ec8b21 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 11 Nov 2020 09:21:26 -0700
Subject: [PATCH 471/882] usage example location

---
 src/libutil/error.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index d1b6d82bb..d42781311 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -38,7 +38,7 @@ namespace nix {
    ErrorInfo structs are sent to the logger as part of an exception, or directly with the
    logError or logWarning macros.
 
-   See the error-demo.cc program for usage examples.
+   See libutil/tests/logging.cc for usage examples.
 
  */
 

From 8abb80a478116b10bf37162c71f602262de412a9 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Thu, 22 Oct 2020 23:59:01 -0500
Subject: [PATCH 472/882] Print built derivations as json for build

Add --json option to nix build to allow machine readable output on
stdout with all built derivations

Fixes #1930
---
 src/nix/build.cc        |  6 +++++-
 src/nix/installables.cc | 27 +++++++++++++++++++++++++++
 src/nix/installables.hh |  5 +++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 65708e98b..67be4024b 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -5,9 +5,11 @@
 #include "store-api.hh"
 #include "local-fs-store.hh"
 
+#include <nlohmann/json.hpp>
+
 using namespace nix;
 
-struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
+struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 {
     Path outLink = "result";
     BuildMode buildMode = bmNormal;
@@ -86,6 +88,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
                     }, buildables[i]);
 
         updateProfile(buildables);
+
+        if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump());
     }
 };
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 7473c9758..f385289e5 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -16,8 +16,35 @@
 #include <regex>
 #include <queue>
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
+nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
+    nlohmann::json res;
+    res["path"] = store->printStorePath(path);
+    return res;
+}
+
+nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
+    nlohmann::json res;
+    res["drvPath"] = store->printStorePath(drvPath);
+    for (const auto& [output, path] : outputs) {
+        res["outputs"][output] = path ? store->printStorePath(*path) : "";
+    }
+    return res;
+}
+
+nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
+    auto res = nlohmann::json::array();
+    for (const Buildable & buildable : buildables) {
+        std::visit([&res, store](const auto & buildable) {
+            res.push_back(buildable.toJSON(store));
+        }, buildable);
+    }
+    return res;
+}
+
 void completeFlakeInputPath(
     ref<EvalState> evalState,
     const FlakeRef & flakeRef,
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index c7c2f8981..f37b3f829 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -7,6 +7,8 @@
 
 #include <optional>
 
+#include <nlohmann/json_fwd.hpp>
+
 namespace nix {
 
 struct DrvInfo;
@@ -16,11 +18,13 @@ namespace eval_cache { class EvalCache; class AttrCursor; }
 
 struct BuildableOpaque {
     StorePath path;
+    nlohmann::json toJSON(ref<Store> store) const;
 };
 
 struct BuildableFromDrv {
     StorePath drvPath;
     std::map<std::string, std::optional<StorePath>> outputs;
+    nlohmann::json toJSON(ref<Store> store) const;
 };
 
 typedef std::variant<
@@ -29,6 +33,7 @@ typedef std::variant<
 > Buildable;
 
 typedef std::vector<Buildable> Buildables;
+nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
 
 struct App
 {

From d52b12c0a53f88b8ea9238d604f293b54c8ae51a Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Fri, 23 Oct 2020 08:20:38 -0500
Subject: [PATCH 473/882] Test nix build --json

---
 tests/build.sh | 12 ++++++++++++
 tests/local.mk |  3 ++-
 2 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 tests/build.sh

diff --git a/tests/build.sh b/tests/build.sh
new file mode 100644
index 000000000..aa54b88eb
--- /dev/null
+++ b/tests/build.sh
@@ -0,0 +1,12 @@
+source common.sh
+
+expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
+nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs.first | match(".*multiple-outputs-a-first")) and
+    (.outputs.second | match(".*multiple-outputs-a-second")))
+  and (.[1] |
+    (.drvPath | match(".*multiple-outputs-b.drv")) and
+    (.outputs.out | match(".*multiple-outputs-b")))
+'
diff --git a/tests/local.mk b/tests/local.mk
index a1929f96d..ce94ec80e 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -35,7 +35,8 @@ nix_tests = \
   recursive.sh \
   describe-stores.sh \
   flakes.sh \
-  content-addressed.sh
+  content-addressed.sh \
+  build.sh
   # parallel.sh
   # build-remote-content-addressed-fixed.sh \
 

From 3edfe6090e9e15b205c21b19530607cbdcbbbe7a Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 11 Nov 2020 09:29:32 -0700
Subject: [PATCH 474/882] missing argument error

---
 src/libexpr/eval.cc    | 12 ++++++++++--
 src/libexpr/nixexpr.hh |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2b697882b..cf9f6c543 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -660,6 +660,14 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
     });
 }
 
+LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1))
+{
+    throw MissingArgumentError({
+        .hint = hintfmt(s, s1),
+        .errPos = pos
+    });
+}
+
 LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
 {
     e.addTrace(std::nullopt, s, s2);
@@ -1365,7 +1373,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             if (j != args.end()) {
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
-                throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%')
+                throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%')
 
 An 'auto-call' is when a nix expression is evaluated without any external arguments.
 If that nix expression is a function, and that function's arguments all have default
@@ -1373,7 +1381,7 @@ values, then all is well.
 
 But if the function arguments don't have default values, evaluation fails.
 
-The classic case for this error is evaluating a nix file with nix-build that expects
+The classic case for this error is evaluating a nix file that expects
 to be evaluated by callPackage.
   # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.
   # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function.
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index e4cbc660f..bf2cd1f15 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError);
 MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
+MakeError(MissingArgumentError, Error);
 MakeError(RestrictedPathError, Error);
 
 

From 8895fa70a4b05ddebbb5a23ea96464d5e01345fb Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 11 Nov 2020 11:05:21 -0700
Subject: [PATCH 475/882] pare down the error message

---
 src/libexpr/eval.cc | 26 ++++----------------------
 1 file changed, 4 insertions(+), 22 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index cf9f6c543..540bfcf7b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1373,31 +1373,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             if (j != args.end()) {
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
-                throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%')
+                throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
-An 'auto-call' is when a nix expression is evaluated without any external arguments.
-If that nix expression is a function, and that function's arguments all have default
-values, then all is well.
+Nix attempted to evaluate a function as a top level expression; in this case it must have all its 
+arguments supplied either by default values, or passed explicitly with --arg or --argstr.
 
-But if the function arguments don't have default values, evaluation fails.
+https://nixos.org/manual/nix/stable/#ss-functions)", i.name);
 
-The classic case for this error is evaluating a nix file that expects
-to be evaluated by callPackage.
-  # in 'callPackage' format: expression is a function that takes an argument 'stdenv'.
-  # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function.
-  { stdenv }:
-  stdenv.mkDerivation  {
-  ...
-
-  # in 'auto-call' format: nixpkgs is imported explicitly, and used directly.
-  let
-    nixpkgs = import <nixpkgs> {};
-  in
-    nixpkgs.stdenv.mkDerivation {
-  ...
-
-More about callPackage: 
-https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name);
             }
         }
     }

From b327de9c2d5182e5814b2e956631b8794b45999b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 11 Nov 2020 11:09:59 -0700
Subject: [PATCH 476/882] change message

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 540bfcf7b..3667ee6ba 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1375,7 +1375,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             } else if (!i.def) {
                 throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
-Nix attempted to evaluate a function as a top level expression; in this case it must have all its 
+nix attempted to evaluate a function as a top level expression; in this case it must have its
 arguments supplied either by default values, or passed explicitly with --arg or --argstr.
 
 https://nixos.org/manual/nix/stable/#ss-functions)", i.name);

From c4c3c15c19bc448a4797e5d9577539cc14890618 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20H=C3=B6ppner?= <chris@mkaito.net>
Date: Thu, 12 Nov 2020 15:46:08 +0000
Subject: [PATCH 477/882] Fix default nix-path

The default nix-path values for nixpkgs and root channels were
incorrect.
---
 src/libexpr/eval.cc | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 5e3fcf4ac..c6f4d1716 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2104,10 +2104,19 @@ EvalSettings::EvalSettings()
 Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
-    auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
+    auto add = [&](const Path & p, const std::string & s = std::string()) {
+        if (pathExists(p)) {
+            if (s.empty()) {
+                res.push_back(p);
+            } else {
+                res.push_back(s + "=" + p);
+            }
+        }
+    };
+
     add(getHome() + "/.nix-defexpr/channels");
-    add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
-    add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
+    add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
+    add(settings.nixStateDir + "/profiles/per-user/root/channels");
     return res;
 }
 

From ac5081d28035ef8d6a907fc3bff812d534719f74 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 13 Nov 2020 17:49:27 +0100
Subject: [PATCH 478/882] nix-build: Fix #4197 output order regression

---
 src/nix-build/nix-build.cc | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index f60e0706c..74fafd426 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -487,6 +487,7 @@ static void main_nix_build(int argc, char * * argv)
     else {
 
         std::vector<StorePathWithOutputs> pathsToBuild;
+        std::vector<std::pair<StorePath, std::string>> pathsToBuildOrdered;
 
         std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
 
@@ -498,6 +499,7 @@ static void main_nix_build(int argc, char * * argv)
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
             pathsToBuild.push_back({drvPath, {outputName}});
+            pathsToBuildOrdered.push_back({drvPath, {outputName}});
 
             auto i = drvMap.find(drvPath);
             if (i != drvMap.end())
@@ -513,25 +515,23 @@ static void main_nix_build(int argc, char * * argv)
 
         std::vector<StorePath> outPaths;
 
-        for (auto & [drvPath, info] : drvMap) {
-            auto & [counter, wantedOutputs] = info;
+        for (auto & [drvPath, outputName] : pathsToBuildOrdered) {
+            auto & [counter, _wantedOutputs] = drvMap.at({drvPath});
             std::string drvPrefix = outLink;
             if (counter)
                 drvPrefix += fmt("-%d", counter + 1);
 
             auto builtOutputs = store->queryDerivationOutputMap(drvPath);
 
-            for (auto & outputName : wantedOutputs) {
-                auto outputPath = builtOutputs.at(outputName);
+            auto outputPath = builtOutputs.at(outputName);
 
-                if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
-                    std::string symlink = drvPrefix;
-                    if (outputName != "out") symlink += "-" + outputName;
-                    store2->addPermRoot(outputPath, absPath(symlink));
-                }
-
-                outPaths.push_back(outputPath);
+            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+                std::string symlink = drvPrefix;
+                if (outputName != "out") symlink += "-" + outputName;
+                store2->addPermRoot(outputPath, absPath(symlink));
             }
+
+            outPaths.push_back(outputPath);
         }
 
         logger->stop();

From d264da8d968bb3afe29def1f4aff0f91f25e36c6 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 13 Nov 2020 17:49:55 +0100
Subject: [PATCH 479/882] tests: Test #4197 nix-build output order regression

---
 tests/nix-build-examples.nix | 33 +++++++++++++++++++++++++++++++++
 tests/nix-build.sh           | 15 +++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 tests/nix-build-examples.nix

diff --git a/tests/nix-build-examples.nix b/tests/nix-build-examples.nix
new file mode 100644
index 000000000..e54dbbf62
--- /dev/null
+++ b/tests/nix-build-examples.nix
@@ -0,0 +1,33 @@
+with import ./config.nix;
+
+rec {
+
+  input0 = mkDerivation {
+    name = "dependencies-input-0";
+    buildCommand = "mkdir $out; echo foo > $out/bar";
+  };
+
+  input1 = mkDerivation {
+    name = "dependencies-input-1";
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
+  };
+
+  input2 = mkDerivation {
+    name = "dependencies-input-2";
+    buildCommand = ''
+      mkdir $out
+      echo BAR > $out/bar
+      echo ${input0} > $out/input0
+    '';
+  };
+
+  body = mkDerivation {
+    name = "dependencies-top";
+    builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
+    input1 = input1 + "/.";
+    input2 = "${input2}/.";
+    input1_drv = input1;
+    meta.description = "Random test package";
+  };
+
+}
diff --git a/tests/nix-build.sh b/tests/nix-build.sh
index 3123c6da3..44a5a14cd 100644
--- a/tests/nix-build.sh
+++ b/tests/nix-build.sh
@@ -26,3 +26,18 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
 
 outPath2=$(nix-store -r $(nix-instantiate --add-root $TEST_ROOT/indirect dependencies.nix)!out)
 [[ $outPath = $outPath2 ]]
+
+# The order of the paths on stdout must correspond to the -A options
+# https://github.com/NixOS/nix/issues/4197
+
+input0="$(nix-build nix-build-examples.nix -A input0 --no-out-link)"
+input1="$(nix-build nix-build-examples.nix -A input1 --no-out-link)"
+input2="$(nix-build nix-build-examples.nix -A input2 --no-out-link)"
+body="$(nix-build nix-build-examples.nix -A body --no-out-link)"
+
+outPathsA="$(echo $(nix-build nix-build-examples.nix -A input0 -A input1 -A input2 -A body --no-out-link))"
+[[ "$outPathsA" = "$input0 $input1 $input2 $body" ]]
+
+# test a different ordering to make sure it fails, not just in 23 out of 24 permutations
+outPathsB="$(echo $(nix-build nix-build-examples.nix -A body -A input1 -A input2 -A input0 --no-out-link))"
+[[ "$outPathsB" = "$body $input1 $input2 $input0" ]]

From 01db455733899728947a15f8d50942f10432d61e Mon Sep 17 00:00:00 2001
From: Jake Waksbaum <jake.waksbaum@gmail.com>
Date: Mon, 16 Nov 2020 02:35:50 -0500
Subject: [PATCH 480/882] Fix deadlock in nix-store when max-connections=1

This fixes a bug I encountered where `nix-store -qR` will deadlock when
the `--include-outputs` flag is passed and `max-connections=1`.

The deadlock occurs because `RemoteStore::queryDerivationOutputs` takes
the only connection from the connection pool and uses it to check the
daemon version. If the version is new enough, it calls
`Store::queryDerivationOutputs`, which eventually calls
`RemoteStore::queryPartialDerivationOutputMap`, where we take another
connection from the connection pool to check the version again. Because
we still haven't released the connection from the caller, this waits for
a connection to be available, causing a deadlock.

This diff solves the issue by using `getProtocol` to check the protocol
version in the caller `RemoteStore::queryDerivationOutputs`, which
immediately frees the connection back to the pool before returning the
protocol version. That way we've already freed the connection by the
time we call `RemoteStore::queryPartialDerivationOutputMap`.
---
 src/libstore/remote-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b6f70057d..a392f3b8c 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -409,10 +409,10 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
 
 StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 {
-    auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
+    if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
         return Store::queryDerivationOutputs(path);
     }
+    auto conn(getConnection());
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
     return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});

From ef84c780bb901011e090b9f12d293d136193a428 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 16 Nov 2020 16:26:29 +0100
Subject: [PATCH 481/882] filterANSIEscapes(): Handle UTF-8 characters

---
 src/libutil/tests/tests.cc | 10 ++++++++++
 src/libutil/util.cc        | 23 ++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index ffba832d8..35a5d27bb 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -587,4 +587,14 @@ namespace nix {
 
         ASSERT_EQ(filterANSIEscapes(s, true), "foo     bar     baz" );
     }
+
+    TEST(filterANSIEscapes, utf8) {
+        ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba");
+        ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär");
+        ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä");
+        ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó");
+        ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b");
+        ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b");
+    }
+
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 53342b5cb..01ab9111f 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1409,7 +1409,28 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
             i++;
 
         else {
-            t += *i++; w++;
+            w++;
+            // Copy one UTF-8 character.
+            if ((*i & 0xe0) == 0xc0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+            } else if ((*i & 0xf0) == 0xe0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                    t += *i++;
+                    if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+                }
+            } else if ((*i & 0xf8) == 0xf0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                    t += *i++;
+                    if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                        t += *i++;
+                        if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+                    }
+                }
+            } else
+                t += *i++;
         }
     }
 

From 7de21f6664ffd4a66e5ef70058a4c450985a1981 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 13 Nov 2020 17:00:32 +0100
Subject: [PATCH 482/882] Make the sql debug statements more useful

Print the expanded sql query (with the variables bound to their value) rather
than the original one in case of error
---
 src/libstore/sqlite.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 31a1f0cac..f5935ee5c 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -147,14 +147,14 @@ void SQLiteStmt::Use::exec()
     int r = step();
     assert(r != SQLITE_ROW);
     if (r != SQLITE_DONE)
-        throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
+        throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt)));
 }
 
 bool SQLiteStmt::Use::next()
 {
     int r = step();
     if (r != SQLITE_DONE && r != SQLITE_ROW)
-        throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
+        throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt)));
     return r == SQLITE_ROW;
 }
 

From bccff827dc968b08bddda03aadcb3d9cc41c2719 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Nov 2020 13:50:36 +0100
Subject: [PATCH 483/882] Fix deadlock in IFD through the daemon

Fixes #4235.
---
 src/libstore/derivations.cc |  6 ++++--
 tests/remote-store.sh       | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 517ecfaa2..eea129df3 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -463,13 +463,15 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
 {
     auto h = drvHashes.find(drvPath);
     if (h == drvHashes.end()) {
-        assert(store.isValidPath(drvPath));
         // Cache it
         h = drvHashes.insert_or_assign(
             drvPath,
             hashDerivationModulo(
                 store,
-                store.readDerivation(drvPath),
+                parseDerivation(
+                    store,
+                    readFile(store.toRealPath(drvPath)),
+                    Derivation::nameFromPath(drvPath)),
                 false)).first;
     }
     return h->second;
diff --git a/tests/remote-store.sh b/tests/remote-store.sh
index 3a61946f9..f7ae1a2ed 100644
--- a/tests/remote-store.sh
+++ b/tests/remote-store.sh
@@ -7,6 +7,20 @@ nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor
 
 startDaemon
 
+# Test import-from-derivation through the daemon.
+[[ $(nix eval --impure --raw --expr '
+  with import ./config.nix;
+  import (
+    mkDerivation {
+      name = "foo";
+      bla = import ./dependencies.nix;
+      buildCommand = "
+        echo \\\"hi\\\" > $out
+      ";
+    }
+  )
+') = hi ]]
+
 storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh
 
 nix-store --dump-db > $TEST_ROOT/d1

From e6b7c7b79c697f1d8508930964e8c810f04a8963 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Nov 2020 13:58:55 +0100
Subject: [PATCH 484/882] Cleanup

---
 src/libstore/derivations.cc |  5 +----
 src/libstore/local-store.cc | 10 ++--------
 src/libstore/store-api.cc   |  8 ++++++++
 src/libstore/store-api.hh   |  3 +++
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index eea129df3..106ec5daa 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -468,10 +468,7 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
             drvPath,
             hashDerivationModulo(
                 store,
-                parseDerivation(
-                    store,
-                    readFile(store.toRealPath(drvPath)),
-                    Derivation::nameFromPath(drvPath)),
+                store.readInvalidDerivation(drvPath),
                 false)).first;
     }
     return h->second;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e4e404dca..93d073768 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -624,10 +624,7 @@ uint64_t LocalStore::addValidPath(State & state,
        efficiently query whether a path is an output of some
        derivation. */
     if (info.path.isDerivation()) {
-        auto drv = parseDerivation(
-            *this,
-            readFile(Store::toRealPath(info.path)),
-            Derivation::nameFromPath(info.path));
+        auto drv = readInvalidDerivation(info.path);
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -1003,10 +1000,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
                 checkDerivationOutputs(i.path,
-                    parseDerivation(
-                        *this,
-                        readFile(Store::toRealPath(i.path)),
-                        Derivation::nameFromPath(i.path)));
+                    readInvalidDerivation(i.path));
             }
 
         /* Do a topological sort of the paths.  This will throw an
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 83d3a1fa1..7f808f45a 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1007,6 +1007,14 @@ Derivation Store::readDerivation(const StorePath & drvPath)
     }
 }
 
+Derivation Store::readInvalidDerivation(const StorePath & drvPath)
+{
+    return parseDerivation(
+        *this,
+        readFile(Store::toRealPath(drvPath)),
+        Derivation::nameFromPath(drvPath));
+}
+
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f77bc21d1..30c9e7d0a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -611,6 +611,9 @@ public:
     /* Read a derivation (which must already be valid). */
     Derivation readDerivation(const StorePath & drvPath);
 
+    /* Read a derivation from a potentially invalid path. */
+    Derivation readInvalidDerivation(const StorePath & drvPath);
+
     /* Place in `out' the set of all store paths in the file system
        closure of `storePath'; that is, all paths than can be directly
        or indirectly reached from it.  `out' is not cleared.  If

From 3daa256728e29f9059a79b8ecaf9325e52f1704a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Nov 2020 15:26:39 +0100
Subject: [PATCH 485/882] Remove tests.binaryTarball

This test no longer works on Hydra because import-from-derivation is
no longer allowed.
---
 flake.nix | 32 --------------------------------
 1 file changed, 32 deletions(-)

diff --git a/flake.nix b/flake.nix
index 8ff048be3..653cacded 100644
--- a/flake.nix
+++ b/flake.nix
@@ -388,38 +388,6 @@
               inherit (self) overlay;
             });
 
-        # Test whether the binary tarball works in an Ubuntu system.
-        tests.binaryTarball =
-          with nixpkgsFor.x86_64-linux;
-          vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
-            { diskImage = vmTools.diskImages.ubuntu1204x86_64;
-            }
-            ''
-              set -x
-              useradd -m alice
-              su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
-              mkdir /dest-nix
-              mount -o bind /dest-nix /nix # Provide a writable /nix.
-              chown alice /nix
-              su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
-              su - alice -c 'nix-store --verify'
-              su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
-
-              # Check whether 'nix upgrade-nix' works.
-              cat > /tmp/paths.nix <<EOF
-              {
-                x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
-              }
-              EOF
-              su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
-              (! [ -L /home/alice/.profile-1-link ])
-              su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
-
-              mkdir -p $out/nix-support
-              touch $out/nix-support/hydra-build-products
-              umount /nix
-            '');
-
         /*
         # Check whether we can still evaluate all of Nixpkgs.
         tests.evalNixpkgs =

From f89fd0bde7ada12c102a6bf7a1a15cbd45aef0b2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Nov 2020 15:36:20 +0100
Subject: [PATCH 486/882] Remove stray debug statement

This was causing a failure on macOS.

https://hydra.nixos.org/build/130354318
---
 tests/binary-cache.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 6415c9302..e3b3982fe 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -258,9 +258,6 @@ docPath=$(nix-store -q --references $outPath)
 # +---...-multi-output-doc
 
 nix copy --to "file://$cacheDir" $outPath
-( echo $outPath $docPath
-  find $cacheDir
-) >/tmp/blurb
 
 hashpart() {
   basename "$1" | cut -c1-32

From 0fa6d380b2143af68d4c7c6ab55b639516014304 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 18 Nov 2020 11:20:50 +0700
Subject: [PATCH 487/882] fix typo in comment in fetchurl.nix

---
 corepkgs/fetchurl.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix
index a84777f57..02531103b 100644
--- a/corepkgs/fetchurl.nix
+++ b/corepkgs/fetchurl.nix
@@ -1,6 +1,6 @@
 { system ? "" # obsolete
 , url
-, hash ? "" # an SRI ash
+, hash ? "" # an SRI hash
 
 # Legacy hash specification
 , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""

From 2113ae2d856a208350ccbafdc19e8dda322515b8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 19 Nov 2020 16:50:06 +0000
Subject: [PATCH 488/882] Make drv hash modulo memo table thread-safe

Let's get one step closer to the daemon not needing to fork.
---
 src/libexpr/primops.cc      |  7 ++++---
 src/libstore/derivations.cc | 27 +++++++++++++++------------
 src/libstore/derivations.hh |  3 ++-
 3 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 236433ef1..41f06c219 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1132,9 +1132,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
        However, we don't bother doing this for floating CA derivations because
        their "hash modulo" is indeterminate until built. */
-    if (drv.type() != DerivationType::CAFloating)
-        drvHashes.insert_or_assign(drvPath,
-            hashDerivationModulo(*state.store, Derivation(drv), false));
+    if (drv.type() != DerivationType::CAFloating) {
+        auto h = hashDerivationModulo(*state.store, Derivation(drv), false);
+        drvHashes.lock()->insert_or_assign(drvPath, h);
+    }
 
     state.mkAttrs(v, 1 + drv.outputs.size());
     mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 106ec5daa..231ca26c2 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -451,7 +451,7 @@ DerivationType BasicDerivation::type() const
 }
 
 
-DrvHashes drvHashes;
+Sync<DrvHashes> drvHashes;
 
 /* pathDerivationModulo and hashDerivationModulo are mutually recursive
  */
@@ -459,19 +459,22 @@ DrvHashes drvHashes;
 /* Look up the derivation by value and memoize the
    `hashDerivationModulo` call.
  */
-static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
+static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
 {
-    auto h = drvHashes.find(drvPath);
-    if (h == drvHashes.end()) {
-        // Cache it
-        h = drvHashes.insert_or_assign(
-            drvPath,
-            hashDerivationModulo(
-                store,
-                store.readInvalidDerivation(drvPath),
-                false)).first;
+    {
+        auto hashes = drvHashes.lock();
+        auto h = hashes->find(drvPath);
+        if (h != hashes->end()) {
+            return h->second;
+        }
     }
-    return h->second;
+    auto h = hashDerivationModulo(
+        store,
+        store.readInvalidDerivation(drvPath),
+        false);
+    // Cache it
+    drvHashes.lock()->insert_or_assign(drvPath, h);
+    return h;
 }
 
 /* See the header for interface details. These are the implementation details.
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index f9ba935e6..b966d6d90 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -210,7 +210,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
 /* Memoisation of hashDerivationModulo(). */
 typedef std::map<StorePath, DrvHashModulo> DrvHashes;
 
-extern DrvHashes drvHashes; // FIXME: global, not thread-safe
+// FIXME: global, though at least thread-safe.
+extern Sync<DrvHashes> drvHashes;
 
 /* Memoisation of `readDerivation(..).resove()`. */
 typedef std::map<

From 0327580e5485143acb6eb7c8c515e3e179f1e3fe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Nov 2020 20:30:41 +0100
Subject: [PATCH 489/882] Fix assertion failure in LockFile::LockFile()

Fixes #4241.
---
 src/libexpr/flake/lockfile.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index 8e2f7131f..6089d1363 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -78,7 +78,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
     {
         if (jsonNode.find("inputs") == jsonNode.end()) return;
         for (auto & i : jsonNode["inputs"].items()) {
-            if (i.value().is_array()) {
+            if (i.value().is_array()) { // FIXME: remove, obsolete
                 InputPath path;
                 for (auto & j : i.value())
                     path.push_back(j);
@@ -87,10 +87,13 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
                 std::string inputKey = i.value();
                 auto k = nodeMap.find(inputKey);
                 if (k == nodeMap.end()) {
-                    auto jsonNode2 = json["nodes"][inputKey];
-                    auto input = std::make_shared<LockedNode>(jsonNode2);
+                    auto nodes = json["nodes"];
+                    auto jsonNode2 = nodes.find(inputKey);
+                    if (jsonNode2 == nodes.end())
+                        throw Error("lock file references missing node '%s'", inputKey);
+                    auto input = std::make_shared<LockedNode>(*jsonNode2);
                     k = nodeMap.insert_or_assign(inputKey, input).first;
-                    getInputs(*input, jsonNode2);
+                    getInputs(*input, *jsonNode2);
                 }
                 if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
                     node.inputs.insert_or_assign(i.key(), child);

From 4dcb183af31d5cb33b6ef8e581e77d1c892a58b9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Nov 2020 20:59:36 +0100
Subject: [PATCH 490/882] AttrCursor::getStringWithContext(): Force
 re-evaluation if the cached context is not valid

Fixes #4236.
---
 src/libexpr/eval-cache.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 381344b40..7b025be23 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -525,8 +525,17 @@ string_t AttrCursor::getStringWithContext()
             cachedValue = root->db->getAttr(getKey(), root->state.symbols);
         if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
-                debug("using cached string attribute '%s'", getAttrPathStr());
-                return *s;
+                bool valid = true;
+                for (auto & c : s->second) {
+                    if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+                        valid = false;
+                        break;
+                    }
+                }
+                if (valid) {
+                    debug("using cached string attribute '%s'", getAttrPathStr());
+                    return *s;
+                }
             } else
                 throw TypeError("'%s' is not a string", getAttrPathStr());
         }

From c3bad73e27a5bda3f8f8c768cec818c52b0f95e3 Mon Sep 17 00:00:00 2001
From: Wil Taylor <cert@wiltaylor.dev>
Date: Sat, 21 Nov 2020 14:28:49 +1000
Subject: [PATCH 491/882] Added switch

---
 src/nix/bundle.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 2d0a0b6ea..c59018726 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -11,6 +11,7 @@ struct CmdBundle : InstallableCommand
 {
     std::string bundler = "github:matthewbauer/nix-bundle";
     std::optional<Path> outLink;
+    bool skipReferenceCheck = false;
 
     CmdBundle()
     {
@@ -32,6 +33,12 @@ struct CmdBundle : InstallableCommand
             .handler = {&outLink},
             .completer = completePath
         });
+
+        addFlag({
+            .longName = "skip-refcheck",
+            .description = "Skip checking of references in bundle.",
+            .handler = {&skipReferenceCheck, true},
+            });
     }
 
     std::string description() override
@@ -117,7 +124,7 @@ struct CmdBundle : InstallableCommand
         auto outPathS = store->printStorePath(outPath);
 
         auto info = store->queryPathInfo(outPath);
-        if (!info->references.empty())
+        if (!info->references.empty() && !skipReferenceCheck)
             throw Error("'%s' has references; a bundler must not leave any references", outPathS);
 
         if (!outLink)

From 233b61d3d6482544c35b9d340240bf3260acff13 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Fri, 6 Nov 2020 11:53:00 +0100
Subject: [PATCH 492/882] installer: simplify the per-build installation

The goal is to allow the installation and testing of arbitrary Nix
versions. Extend the base installer to accept a `--tarball-url-prefix
<url>` to change where the Nix tarball is getting downloaded from.

Once this is merged it should allow to:
1. Pick an evaluation at https://hydra.nixos.org/jobset/nix/master that
   looks healthy
2. Select the installedScript build and find the store path.

Now equipped with all of this, use an instance of nar-serve to fetch the
install script and release tarballs:

    curl -sfL https://nar-serve.numtide.com/nix/store/rkv4yh7pym941bhj0849zqdkg2546bdv-installer-script/install \
      | sh --tarball-url-prefix https://nar-serve.numtide.com/nix/store

Or with cachix, strip the /nix/store and derivation name and then:

    curl -sfL https://mycache.cachix.org/serve/rkv4yh7pym941bhj0849zqdkg2546bdv/install \
      | sh --tarball-url-prefix https://mycache.cachix.org/serve

Fixes #4047
---
 flake.nix                           | 19 +++++++++++-
 scripts/install-nix-from-closure.sh |  3 ++
 scripts/install.in                  | 46 ++++++++++++++++++++++++-----
 3 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/flake.nix b/flake.nix
index 653cacded..7311a2471 100644
--- a/flake.nix
+++ b/flake.nix
@@ -324,9 +324,26 @@
             ''
               mkdir -p $out/nix-support
 
+              # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
+              # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+              tarballPath() {
+                # Remove the store prefix
+                local path=''${1#${builtins.storeDir}/}
+                # Get the path relative to the derivation root
+                local rest=''${path#*/}
+                # Get the derivation hash
+                local drvHash=''${path%%-*}
+                echo "$drvHash/$rest"
+              }
+
               substitute ${./scripts/install.in} $out/install \
                 ${pkgs.lib.concatMapStrings
-                  (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
+                  (system:
+                    '' \
+                    --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                    --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                    ''
+                  )
                   [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
                 } \
                 --replace '@nixVersion@' ${version}
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 2de41a6a4..6352a8fac 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -98,6 +98,9 @@ while [ $# -gt 0 ]; do
                 echo ""
                 echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
                 echo ""
+                if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then
+                    echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from."
+                fi
             ) >&2
 
             # darwin and Catalina+
diff --git a/scripts/install.in b/scripts/install.in
index 9a281d776..7c3b795cc 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -25,18 +25,47 @@ require_util() {
 }
 
 case "$(uname -s).$(uname -m)" in
-    Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;;
-    Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
-    Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
-    Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
-    # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;;
-    Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
+    Linux.x86_64)
+        hash=@tarballHash_x86_64-linux@
+        path=@tarballPath_x86_64-linux@
+        system=x86_64-linux
+        ;;
+    Linux.i?86)
+        hash=@tarballHash_i686-linux@
+        path=@tarballPath_i686-linux@
+        system=i686-linux
+        ;;
+    Linux.aarch64)
+        hash=@tarballHash_aarch64-linux@
+        path=@tarballPath_aarch64-linux@
+        system=aarch64-linux
+        ;;
+    Darwin.x86_64)
+        hash=@tarballHash_x86_64-darwin@
+        path=@tarballPath_x86_64-darwin@
+        system=x86_64-darwin
+        ;;
+    Darwin.arm64)
+        hash=@binaryTarball_x86_64-darwin@
+        path=@tarballPath_x86_64-darwin@
+        # eventually maybe: arm64-darwin
+        system=x86_64-darwin
+        ;;
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+# Use this command-line option to fetch the tarballs using nar-serve or Cachix
+if "${1:---tarball-url-prefix}"; then
+    if [ -z "${2:-}" ]; then
+        oops "missing argument for --tarball-url-prefix"
+    fi
+    url=${2}/${path}
+    shift 2
+else
+    url=https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz
+fi
 
-tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
+tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
@@ -68,6 +97,7 @@ tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
 script=$(echo "$unpack"/*/install)
 
 [ -e "$script" ] || oops "installation script is missing from the binary tarball!"
+export INVOKED_FROM_INSTALL_IN=1
 "$script" "$@"
 
 } # End of wrapping

From df83b6df68a7a7d71747c1d6cfb98f6b4ee732f0 Mon Sep 17 00:00:00 2001
From: Kai Wohlfahrt <kai.wohlfahrt@gmail.com>
Date: Sat, 21 Nov 2020 22:06:15 +0000
Subject: [PATCH 493/882] Return signatures in Perl path info

---
 perl/lib/Nix/Store.xs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 599921151..9e3b7d389 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -110,10 +110,14 @@ SV * queryPathInfo(char * path, int base32)
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
-            AV * arr = newAV();
+            AV * refs = newAV();
             for (auto & i : info->references)
-                av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
-            XPUSHs(sv_2mortal(newRV((SV *) arr)));
+                av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0));
+            XPUSHs(sv_2mortal(newRV((SV *) refs)));
+            AV * sigs = newAV();
+            for (auto & i : info->sigs)
+                av_push(sigs, newSVpv(i.c_str(), 0));
+            XPUSHs(sv_2mortal(newRV((SV *) sigs)));
         } catch (Error & e) {
             croak("%s", e.what());
         }

From 07603890d2867907905ba411cee550390d868936 Mon Sep 17 00:00:00 2001
From: Wil Taylor <cert@wiltaylor.dev>
Date: Mon, 23 Nov 2020 21:19:40 +1000
Subject: [PATCH 494/882] Removed reference check from bundler command

---
 src/nix/bundle.cc | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index c59018726..eddd82f40 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -11,7 +11,6 @@ struct CmdBundle : InstallableCommand
 {
     std::string bundler = "github:matthewbauer/nix-bundle";
     std::optional<Path> outLink;
-    bool skipReferenceCheck = false;
 
     CmdBundle()
     {
@@ -34,11 +33,6 @@ struct CmdBundle : InstallableCommand
             .completer = completePath
         });
 
-        addFlag({
-            .longName = "skip-refcheck",
-            .description = "Skip checking of references in bundle.",
-            .handler = {&skipReferenceCheck, true},
-            });
     }
 
     std::string description() override
@@ -123,10 +117,6 @@ struct CmdBundle : InstallableCommand
 
         auto outPathS = store->printStorePath(outPath);
 
-        auto info = store->queryPathInfo(outPath);
-        if (!info->references.empty() && !skipReferenceCheck)
-            throw Error("'%s' has references; a bundler must not leave any references", outPathS);
-
         if (!outLink)
             outLink = baseNameOf(app.program);
 

From 226116f48272dbe25bb9d3b8fa138cf6b10ec8ef Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <git@lukegb.com>
Date: Mon, 23 Nov 2020 16:12:33 +0000
Subject: [PATCH 495/882] fetchMercurial: set HGPLAIN when invoking hg

Without setting HGPLAIN, the user's environment leaks into
hg invocations, which means that the output may not be in the
expected format.

HGPLAIN is the Mercurial-recommended solution for this in that
it's intended for uses by scripts and programs which are looking
to parse Mercurial's output in a consistent manner.
---
 src/libfetchers/mercurial.cc | 54 ++++++++++++++++++++++++++++--------
 tests/fetchMercurial.sh      |  3 ++
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 41f60c45c..07a51059d 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -11,6 +11,36 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
+namespace {
+
+RunOptions hgOptions(const Strings & args) {
+	RunOptions opts("hg", args);
+	opts.searchPath = true;
+
+	auto env = getEnv();
+	// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+	env["HGPLAIN"] = "";
+	opts.environment = env;
+
+	return opts;
+}
+
+// runProgram wrapper that uses hgOptions instead of stock RunOptions.
+string runHg(const Strings & args, const std::optional<std::string> & input = {})
+{
+	RunOptions opts = hgOptions(args);
+	opts.input = input;
+
+	auto res = runProgram(opts);
+
+	if (!statusOk(res.first))
+		throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
+
+	return res.second;
+}
+
+}
+
 struct MercurialInputScheme : InputScheme
 {
     std::optional<Input> inputFromURL(const ParsedURL & url) override
@@ -100,11 +130,11 @@ struct MercurialInputScheme : InputScheme
         assert(sourcePath);
 
         // FIXME: shut up if file is already tracked.
-        runProgram("hg", true,
+        runHg(
             { "add", *sourcePath + "/" + std::string(file) });
 
         if (commitMsg)
-            runProgram("hg", true,
+            runHg(
                 { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
     }
 
@@ -130,7 +160,7 @@ struct MercurialInputScheme : InputScheme
 
         if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
 
-            bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+            bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
 
             if (!clean) {
 
@@ -143,10 +173,10 @@ struct MercurialInputScheme : InputScheme
                 if (settings.warnDirty)
                     warn("Mercurial tree '%s' is unclean", actualUrl);
 
-                input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
+                input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
 
                 auto files = tokenizeString<std::set<std::string>>(
-                    runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+                    runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
 
                 PathFilter filter = [&](const Path & p) -> bool {
                     assert(hasPrefix(p, actualUrl));
@@ -224,33 +254,33 @@ struct MercurialInputScheme : InputScheme
         if (!(input.getRev()
                 && pathExists(cacheDir)
                 && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
+                    hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
                     .killStderr(true)).second == "1"))
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
 
             if (pathExists(cacheDir)) {
                 try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                    runHg({ "pull", "-R", cacheDir, "--", actualUrl });
                 }
                 catch (ExecError & e) {
                     string transJournal = cacheDir + "/.hg/store/journal";
                     /* hg throws "abandoned transaction" error only if this file exists */
                     if (pathExists(transJournal)) {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                        runHg({ "recover", "-R", cacheDir });
+                        runHg({ "pull", "-R", cacheDir, "--", actualUrl });
                     } else {
                         throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
                     }
                 }
             } else {
                 createDirs(dirOf(cacheDir));
-                runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
+                runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir });
             }
         }
 
         auto tokens = tokenizeString<std::vector<std::string>>(
-            runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
+            runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
         assert(tokens.size() == 3);
 
         input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
@@ -263,7 +293,7 @@ struct MercurialInputScheme : InputScheme
         Path tmpDir = createTempDir();
         AutoDelete delTmpDir(tmpDir, true);
 
-        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
+        runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index af8ef8d5b..d8a4e09d2 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -15,6 +15,9 @@ hg init $repo
 echo '[ui]' >> $repo/.hg/hgrc
 echo 'username = Foobar <foobar@example.org>' >> $repo/.hg/hgrc
 
+# Set ui.tweakdefaults to ensure HGPLAIN is being set.
+echo 'tweakdefaults = True' >> $repo/.hg/hgrc
+
 echo utrecht > $repo/hello
 touch $repo/.hgignore
 hg add --cwd $repo hello .hgignore

From 5b0790355fe10b5cdc2468928a7cb4703cd0861a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabian=20M=C3=B6ller?= <fabianm88@gmail.com>
Date: Mon, 23 Nov 2020 17:40:17 +0100
Subject: [PATCH 496/882] Fix macOS sandbox build

Since c4c3c15c19bc448a4797e5d9577539cc14890618 (#4251) building Nix for
macOS with sandboxing fails:
```
getting status of /nix/var/nix/profiles/per-user/root/channels/nixpkgs: Operation not permitted
```

This happens, because `EvalSettings::getDefaultNixPath` tries to access
paths outside the sandbox. Since the state-dir is not required for
doc generation, it is set to the dummy folder. This needs to be done
for all nix invocations during doc generation, as
`EvalSettings::getDefaultNixPath` is called unconditionally.
---
 doc/manual/local.mk | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 7d9a1a3e8..bb8b3b60a 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -15,7 +15,14 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 dist-files += $(man-pages)
 
-nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
+# Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
+dummy-env = env -i \
+	HOME=/dummy \
+	NIX_CONF_DIR=/dummy \
+	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
+	NIX_STATE_DIR=/dummy
+
+nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
 
 $(d)/%.1: $(d)/src/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
@@ -45,11 +52,11 @@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(bindir)/nix __dump-args > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
@@ -58,7 +65,7 @@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix
 	@mv $@.tmp $@
 
 $(d)/builtins.json: $(bindir)/nix
-	$(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
 	mv $@.tmp $@
 
 # Generate the HTML manual.

From 437189e446e16399d347e4430c4d115b4cf2ddf1 Mon Sep 17 00:00:00 2001
From: Lily Ballard <lily@sb.org>
Date: Tue, 24 Nov 2020 14:12:32 -0800
Subject: [PATCH 497/882] Escape filename given to nix-shell in shebang mode

This prevents spaces or other metacharacters from causing nix-shell to
execute the wrong path.

Fixes #4229.
---
 src/nix-build/nix-build.cc |  4 ++--
 tests/nix-shell.sh         | 18 +++++++++++++++++-
 tests/shell.nix            |  2 +-
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 74fafd426..38048da52 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -217,9 +217,9 @@ static void main_nix_build(int argc, char * * argv)
                 // read the shebang to understand which packages to read from. Since
                 // this is handled via nix-shell -p, we wrap our ruby script execution
                 // in ruby -e 'load' which ignores the shebangs.
-                envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str();
+                envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str();
             } else {
-                envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str();
+                envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str();
             }
         }
 
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 7b2be650a..4775bafb9 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -47,6 +47,14 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh
 output=$($TEST_ROOT/shell.shebang.sh abc def)
 [ "$output" = "foo bar abc def" ]
 
+# Test nix-shell shebang mode again with metacharacters in the filename.
+# First word of filename is chosen to not match any file in the test root.
+sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
+chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
+
+output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
+[ "$output" = "foo bar abc def" ]
+
 # Test nix-shell shebang mode for ruby
 # This uses a fake interpreter that returns the arguments passed
 # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
@@ -54,7 +62,15 @@ sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shel
 chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
-[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
+[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ]
+
+# Test nix-shell shebang mode for ruby again with metacharacters in the filename.
+# Note: fake interpreter only space-separates args without adding escapes to its output.
+sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
+chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
+
+output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
+[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
 
 # Test 'nix develop'.
 nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
diff --git a/tests/shell.nix b/tests/shell.nix
index 6ce59b416..24ebcc04c 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -50,7 +50,7 @@ let pkgs = rec {
   # ruby "interpreter" that outputs "$@"
   ruby = runCommand "ruby" {} ''
     mkdir -p $out/bin
-    echo 'printf -- "$*"' > $out/bin/ruby
+    echo 'printf %s "$*"' > $out/bin/ruby
     chmod a+rx $out/bin/ruby
   '';
 

From 13c557fe823549db75c3dc24c99c46e1c4e1378e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 25 Nov 2020 11:20:03 +0100
Subject: [PATCH 498/882] fix the hash rewriting for ca-derivations

---
 src/libstore/build/derivation-goal.cc | 14 ++++++++++++++
 tests/content-addressed.nix           |  9 ++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index bf2bad62c..0e4504857 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -3121,6 +3121,20 @@ void DerivationGoal::registerOutputs()
             newInfo0.references = refs.second;
             if (refs.first)
                 newInfo0.references.insert(newInfo0.path);
+            if (scratchPath != newInfo0.path) {
+                // Also rewrite the output path
+                auto source = sinkToSource([&](Sink & nextSink) {
+                    StringSink sink;
+                    dumpPath(actualPath, sink);
+                    RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
+                    rsink2((unsigned char *) sink.s->data(), sink.s->size());
+                    rsink2.flush();
+                });
+                Path tmpPath = actualPath + ".tmp";
+                restorePath(tmpPath, *source);
+                deletePath(actualPath);
+                movePath(tmpPath, actualPath);
+            }
 
             assert(newInfo0.ca);
             return newInfo0;
diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 8ca96d4bf..985220f48 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -16,14 +16,16 @@ rec {
   };
   rootCA = mkDerivation {
     name = "rootCA";
-    outputs = [ "out" "dev" ];
+    outputs = [ "out" "dev" "foo"];
     buildCommand = ''
       echo "building a CA derivation"
       echo "The seed is ${toString seed}"
       mkdir -p $out
       echo ${rootLegacy}/hello > $out/dep
-      # test symlink at root
+      ln -s $out $out/self
+      # test symlinks at root
       ln -s $out $dev
+      ln -s $out $foo
     '';
     __contentAddressed = true;
     outputHashMode = "recursive";
@@ -34,7 +36,8 @@ rec {
     buildCommand = ''
       echo "building a dependent derivation"
       mkdir -p $out
-      echo ${rootCA}/hello > $out/dep
+      cat ${rootCA}/self/dep
+      echo ${rootCA}/self/dep > $out/dep
     '';
     __contentAddressed = true;
     outputHashMode = "recursive";

From 0287f8305790a87b128ce09d7d3fa0de7104673c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 26 Nov 2020 12:34:43 +0100
Subject: [PATCH 499/882] Ask for confirmation before allowing flake Nix
 configuration settings

---
 src/libexpr/flake/flake.cc  | 69 ++++++++++++++++++++-----------------
 src/libexpr/flake/flake.hh  |  2 +-
 src/libmain/progress-bar.cc | 11 ++++++
 src/libutil/logging.hh      |  3 ++
 4 files changed, 53 insertions(+), 32 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index bdcf63c21..453d219dc 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -233,28 +233,28 @@ static Flake getFlake(
     if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
         expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos);
 
-        for (auto & option : *nixConfig->value->attrs) {
-            forceTrivialValue(state, *option.value, *option.pos);
-            if (option.value->type == tString)
-                flake.config.options.insert({option.name, state.forceStringNoCtx(*option.value, *option.pos)});
-            else if (option.value->type == tInt)
-                flake.config.options.insert({option.name, state.forceInt(*option.value, *option.pos)});
-            else if (option.value->type == tBool)
-                flake.config.options.insert({option.name, state.forceBool(*option.value, *option.pos)});
-            else if (option.value->isList()) {
+        for (auto & setting : *nixConfig->value->attrs) {
+            forceTrivialValue(state, *setting.value, *setting.pos);
+            if (setting.value->type == tString)
+                flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
+            else if (setting.value->type == tInt)
+                flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
+            else if (setting.value->type == tBool)
+                flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
+            else if (setting.value->isList()) {
                 std::vector<std::string> ss;
-                for (unsigned int n = 0; n < option.value->listSize(); ++n) {
-                    auto elem = option.value->listElems()[n];
+                for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
+                    auto elem = setting.value->listElems()[n];
                     if (elem->type != tString)
-                        throw TypeError("list element in flake configuration option '%s' is %s while a string is expected",
-                            option.name, showType(*option.value));
-                    ss.push_back(state.forceStringNoCtx(*elem, *option.pos));
+                        throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
+                            setting.name, showType(*setting.value));
+                    ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
                 }
-                flake.config.options.insert({option.name, ss});
+                flake.config.settings.insert({setting.name, ss});
             }
             else
-                throw TypeError("flake configuration option '%s' is %s",
-                    option.name, showType(*option.value));
+                throw TypeError("flake configuration setting '%s' is %s",
+                    setting.name, showType(*setting.value));
         }
     }
 
@@ -637,27 +637,34 @@ Flake::~Flake() { }
 
 void ConfigFile::apply()
 {
-    for (auto & [name, value] : options) {
-        // FIXME: support 'trusted-public-keys' (and other options), but make it TOFU.
-        if (name != "bash-prompt-suffix" &&
-            name != "bash-prompt" &&
-            name != "substituters" &&
-            name != "extra-substituters")
-        {
-            warn("ignoring untrusted flake configuration option '%s'", name);
-            continue;
-        }
+    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
+
+    for (auto & [name, value] : settings) {
+
+        auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
+
         // FIXME: Move into libutil/config.cc.
+        std::string valueS;
         if (auto s = std::get_if<std::string>(&value))
-            globalConfig.set(name, *s);
+            valueS = *s;
         else if (auto n = std::get_if<int64_t>(&value))
-            globalConfig.set(name, fmt("%d", n));
+            valueS = fmt("%d", n);
         else if (auto b = std::get_if<Explicit<bool>>(&value))
-            globalConfig.set(name, b->t ? "true" : "false");
+            valueS = b->t ? "true" : "false";
         else if (auto ss = std::get_if<std::vector<std::string>>(&value))
-            globalConfig.set(name, concatStringsSep(" ", *ss)); // FIXME: evil
+            valueS = concatStringsSep(" ", *ss); // FIXME: evil
         else
             assert(false);
+
+        if (!whitelist.count(baseName)) {
+            // FIXME: filter ANSI escapes, newlines, \r, etc.
+            if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '%s' (y/N)?", name, valueS)).value_or('n')) != 'y') {
+                warn("ignoring untrusted flake configuration setting '%s'", name);
+                continue;
+            }
+        }
+
+        globalConfig.set(name, valueS);
     }
 }
 
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 7eebd9044..65ed1ad0a 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -51,7 +51,7 @@ struct ConfigFile
 {
     using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
 
-    std::map<std::string, ConfigValue> options;
+    std::map<std::string, ConfigValue> settings;
 
     void apply();
 };
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 07b45b3b5..0e5432fca 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -466,6 +466,17 @@ public:
             Logger::writeToStdout(s);
         }
     }
+
+    std::optional<char> ask(std::string_view msg) override
+    {
+        auto state(state_.lock());
+        if (!state->active || !isatty(STDIN_FILENO)) return {};
+        std::cerr << fmt("\r\e[K%s ", msg);
+        auto s = trim(readLine(STDIN_FILENO));
+        if (s.size() != 1) return {};
+        draw(*state);
+        return s[0];
+    }
 };
 
 Logger * makeProgressBar(bool printBuildLogs)
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index e3fe613e8..cd0cb64c5 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -106,6 +106,9 @@ public:
         formatHelper(f, args...);
         writeToStdout(f.str());
     }
+
+    virtual std::optional<char> ask(std::string_view s)
+    { return {}; }
 };
 
 ActivityId getCurActivity();

From 9a586e34ac3ae37bfd18f4e82af26df938ab9d96 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 26 Nov 2020 13:11:07 +0100
Subject: [PATCH 500/882] Record trusted/untrusted settings in
 ~/.local/share/nix

---
 src/libexpr/flake/flake.cc | 47 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 453d219dc..0eadd94db 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -6,6 +6,8 @@
 #include "fetchers.hh"
 #include "finally.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 using namespace flake;
@@ -635,6 +637,27 @@ Fingerprint LockedFlake::getFingerprint() const
 
 Flake::~Flake() { }
 
+// setting name -> setting value -> allow or ignore.
+typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
+
+Path trustedListPath()
+{
+    return getDataDir() + "/nix/trusted-settings.json";
+}
+
+static TrustedList readTrustedList()
+{
+    auto path = trustedListPath();
+    if (!pathExists(path)) return {};
+    auto json = nlohmann::json::parse(readFile(path));
+    return json;
+}
+
+static void writeTrustedList(const TrustedList & trustedList)
+{
+    writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
+}
+
 void ConfigFile::apply()
 {
     std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
@@ -657,8 +680,28 @@ void ConfigFile::apply()
             assert(false);
 
         if (!whitelist.count(baseName)) {
-            // FIXME: filter ANSI escapes, newlines, \r, etc.
-            if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '%s' (y/N)?", name, valueS)).value_or('n')) != 'y') {
+            auto trustedList = readTrustedList();
+
+            bool trusted = false;
+
+            if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+                trusted = *saved;
+            } else {
+                // FIXME: filter ANSI escapes, newlines, \r, etc.
+                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
+                    if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
+                        trustedList[name][valueS] = false;
+                        writeTrustedList(trustedList);
+                    }
+                } else {
+                    if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
+                        trustedList[name][valueS] = trusted = true;
+                        writeTrustedList(trustedList);
+                    }
+                }
+            }
+
+            if (!trusted) {
                 warn("ignoring untrusted flake configuration setting '%s'", name);
                 continue;
             }

From 8252a44e96619b6df0901a1f62e60b3d5951fd12 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 26 Nov 2020 13:16:36 +0100
Subject: [PATCH 501/882] Move to separate file

---
 src/libexpr/flake/config.cc | 81 +++++++++++++++++++++++++++++++++++++
 src/libexpr/flake/flake.cc  | 76 ----------------------------------
 2 files changed, 81 insertions(+), 76 deletions(-)
 create mode 100644 src/libexpr/flake/config.cc

diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc
new file mode 100644
index 000000000..63566131e
--- /dev/null
+++ b/src/libexpr/flake/config.cc
@@ -0,0 +1,81 @@
+#include "flake.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::flake {
+
+// setting name -> setting value -> allow or ignore.
+typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
+
+Path trustedListPath()
+{
+    return getDataDir() + "/nix/trusted-settings.json";
+}
+
+static TrustedList readTrustedList()
+{
+    auto path = trustedListPath();
+    if (!pathExists(path)) return {};
+    auto json = nlohmann::json::parse(readFile(path));
+    return json;
+}
+
+static void writeTrustedList(const TrustedList & trustedList)
+{
+    writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
+}
+
+void ConfigFile::apply()
+{
+    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
+
+    for (auto & [name, value] : settings) {
+
+        auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
+
+        // FIXME: Move into libutil/config.cc.
+        std::string valueS;
+        if (auto s = std::get_if<std::string>(&value))
+            valueS = *s;
+        else if (auto n = std::get_if<int64_t>(&value))
+            valueS = fmt("%d", n);
+        else if (auto b = std::get_if<Explicit<bool>>(&value))
+            valueS = b->t ? "true" : "false";
+        else if (auto ss = std::get_if<std::vector<std::string>>(&value))
+            valueS = concatStringsSep(" ", *ss); // FIXME: evil
+        else
+            assert(false);
+
+        if (!whitelist.count(baseName)) {
+            auto trustedList = readTrustedList();
+
+            bool trusted = false;
+
+            if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+                trusted = *saved;
+            } else {
+                // FIXME: filter ANSI escapes, newlines, \r, etc.
+                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
+                    if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
+                        trustedList[name][valueS] = false;
+                        writeTrustedList(trustedList);
+                    }
+                } else {
+                    if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
+                        trustedList[name][valueS] = trusted = true;
+                        writeTrustedList(trustedList);
+                    }
+                }
+            }
+
+            if (!trusted) {
+                warn("ignoring untrusted flake configuration setting '%s'", name);
+                continue;
+            }
+        }
+
+        globalConfig.set(name, valueS);
+    }
+}
+
+}
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 0eadd94db..3e866e1f9 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -6,8 +6,6 @@
 #include "fetchers.hh"
 #include "finally.hh"
 
-#include <nlohmann/json.hpp>
-
 namespace nix {
 
 using namespace flake;
@@ -637,78 +635,4 @@ Fingerprint LockedFlake::getFingerprint() const
 
 Flake::~Flake() { }
 
-// setting name -> setting value -> allow or ignore.
-typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
-
-Path trustedListPath()
-{
-    return getDataDir() + "/nix/trusted-settings.json";
-}
-
-static TrustedList readTrustedList()
-{
-    auto path = trustedListPath();
-    if (!pathExists(path)) return {};
-    auto json = nlohmann::json::parse(readFile(path));
-    return json;
-}
-
-static void writeTrustedList(const TrustedList & trustedList)
-{
-    writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
-}
-
-void ConfigFile::apply()
-{
-    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
-
-    for (auto & [name, value] : settings) {
-
-        auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
-
-        // FIXME: Move into libutil/config.cc.
-        std::string valueS;
-        if (auto s = std::get_if<std::string>(&value))
-            valueS = *s;
-        else if (auto n = std::get_if<int64_t>(&value))
-            valueS = fmt("%d", n);
-        else if (auto b = std::get_if<Explicit<bool>>(&value))
-            valueS = b->t ? "true" : "false";
-        else if (auto ss = std::get_if<std::vector<std::string>>(&value))
-            valueS = concatStringsSep(" ", *ss); // FIXME: evil
-        else
-            assert(false);
-
-        if (!whitelist.count(baseName)) {
-            auto trustedList = readTrustedList();
-
-            bool trusted = false;
-
-            if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
-                trusted = *saved;
-            } else {
-                // FIXME: filter ANSI escapes, newlines, \r, etc.
-                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
-                    if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
-                        trustedList[name][valueS] = false;
-                        writeTrustedList(trustedList);
-                    }
-                } else {
-                    if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
-                        trustedList[name][valueS] = trusted = true;
-                        writeTrustedList(trustedList);
-                    }
-                }
-            }
-
-            if (!trusted) {
-                warn("ignoring untrusted flake configuration setting '%s'", name);
-                continue;
-            }
-        }
-
-        globalConfig.set(name, valueS);
-    }
-}
-
 }

From 1fd13d67e85b8365baed1cfb435870e24a7e6979 Mon Sep 17 00:00:00 2001
From: Dominique Martinet <asmadeus@codewreck.org>
Date: Thu, 26 Nov 2020 14:26:57 +0100
Subject: [PATCH 502/882] archive: disable preallocate-contents by default

using fallocate() to preallocate files space does more harm than good:
 - breaks compression on btrfs
 - has been called "not the right thing to do" by xfs developers
(because delayed allocation that most filesystems implement leads to smarter
allocation than what the filesystem needs to do if we upfront fallocate files)
---
 src/libutil/archive.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index f1479329f..03534abc4 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -27,7 +27,7 @@ struct ArchiveSettings : Config
         #endif
         "use-case-hack",
         "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
-    Setting<bool> preallocateContents{this, true, "preallocate-contents",
+    Setting<bool> preallocateContents{this, false, "preallocate-contents",
         "Whether to preallocate files when writing objects with known size."};
 };
 

From 05d9442f68ba906ae50c12deab63fc1b9836b149 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 26 Nov 2020 21:45:28 +0100
Subject: [PATCH 503/882] builtins.fetchGit: Fix shortRev attribute for dirty
 trees

---
 src/libexpr/primops/fetchTree.cc | 2 +-
 tests/fetchGit.sh                | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index e6f637a43..d094edf92 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -39,7 +39,7 @@ void emitTreeAttrs(
         // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
         auto emptyHash = Hash(htSHA1);
         mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
     }
 
     if (input.getType() == "git")
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index cedd796f7..76390fa59 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -59,6 +59,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath
 
 [[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = ${rev2:0:7} ]]
 
 # Fetching with a explicit hash should succeed.
 path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
@@ -132,6 +133,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath
 path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
 # (check dirty-tree handling was used)
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).shortRev") = 0000000 ]]
 
 # Committing shouldn't change store path, or switch to using 'master'
 git -C $repo commit -m 'Bla5' -a

From 9bd8184f1fb2e91ac4fb7207abe56f2a30a81d97 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 25 Nov 2020 18:20:35 +0100
Subject: [PATCH 504/882] Allow fixed-output derivations to depend on
 (floating) content-addressed ones

Fix an overlook of https://github.com/NixOS/nix/pull/4056
---
 src/libstore/build/derivation-goal.cc |  4 ++--
 tests/content-addressed.nix           | 11 +++++++++++
 tests/content-addressed.sh            |  1 +
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 0e4504857..76c49f92c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -493,8 +493,8 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if ((!fullDrv.inputDrvs.empty() &&
-             fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) {
+        if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
+            || fullDrv.type() == DerivationType::DeferredInputAddressed) {
             /* We are be able to resolve this derivation based on the
                now-known results of dependencies. If so, we become a stub goal
                aliasing that resolved derivation goal */
diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix
index 985220f48..61079176f 100644
--- a/tests/content-addressed.nix
+++ b/tests/content-addressed.nix
@@ -63,4 +63,15 @@ rec {
       echo ${rootCA}/non-ca-hello > $out/dep
     '';
   };
+  dependentFixedOutput = mkDerivation {
+    name = "dependent-fixed-output";
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=";
+    buildCommand = ''
+      cat ${dependentCA}/dep
+      echo foo > $out
+    '';
+
+  };
 }
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index bdab09c86..52f7529b5 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -40,6 +40,7 @@ testCutoff () {
     #testDerivation dependentCA
     testCutoffFor transitivelyDependentCA
     testCutoffFor dependentNonCA
+    testCutoffFor dependentFixedOutput
 }
 
 testGC () {

From e224c16d28be738b260d9d15d0503da3febf99de Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Nov 2020 14:14:11 +0100
Subject: [PATCH 505/882] Macro hygiene

---
 src/libutil/logging.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index cd0cb64c5..ff17056f4 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -178,8 +178,8 @@ extern Verbosity verbosity; /* suppress msgs > this */
    lightweight status messages. */
 #define logErrorInfo(level, errorInfo...) \
     do { \
-        if (level <= nix::verbosity) { \
-            logger->logEI(level, errorInfo); \
+        if ((level) <= nix::verbosity) {     \
+            logger->logEI((level), errorInfo);  \
         } \
     } while (0)
 

From 59276244738ebec32ed0593ee4af615852ff564d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Nov 2020 14:14:57 +0100
Subject: [PATCH 506/882] Lower verbosity for 'Failed to find a machine'
 message

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 9a0e9f08d..8348d8c91 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -201,7 +201,7 @@ static int main_build_remote(int argc, char * * argv)
                             % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
                         }
 
-                        logErrorInfo(lvlInfo, {
+                        logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, {
                               .name = "Remote build",
                               .description = "Failed to find a machine for remote build!",
                               .hint = hint

From 3b7e00ce2215b742d9fdb1b8d4a4d76d349028c7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 18 Nov 2020 14:36:15 +0100
Subject: [PATCH 507/882] Move primeCache() to Worker::run()

We need the missing path info to communicate the worker's remaining
goals to the progress bar.
---
 src/libstore/build/derivation-goal.hh   | 12 +-----------
 src/libstore/build/local-store-build.cc | 22 ++--------------------
 src/libstore/build/substitution-goal.hh |  8 +-------
 src/libstore/build/worker.cc            | 21 ++++++++++++++++++++-
 4 files changed, 24 insertions(+), 39 deletions(-)

diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 4976207e0..8ee0be9e1 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -40,9 +40,8 @@ struct InitialOutput {
     std::optional<InitialOutputStatus> known;
 };
 
-class DerivationGoal : public Goal
+struct DerivationGoal : public Goal
 {
-private:
     /* Whether to use an on-disk .drv file. */
     bool useDerivation;
 
@@ -246,7 +245,6 @@ private:
 
     friend struct RestrictedStore;
 
-public:
     DerivationGoal(const StorePath & drvPath,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
@@ -264,17 +262,11 @@ public:
 
     void work() override;
 
-    StorePath getDrvPath()
-    {
-        return drvPath;
-    }
-
     /* Add wanted outputs to an already existing derivation goal. */
     void addWantedOutputs(const StringSet & outputs);
 
     BuildResult getResult() { return result; }
 
-private:
     /* The states. */
     void getDerivation();
     void loadDerivation();
@@ -318,8 +310,6 @@ private:
     /* Run the builder's process. */
     void runChild();
 
-    friend int childEntry(void *);
-
     /* Check that the derivation outputs all exist and register them
        as valid. */
     void registerOutputs();
diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc
index a05fb5805..c91cda2fd 100644
--- a/src/libstore/build/local-store-build.cc
+++ b/src/libstore/build/local-store-build.cc
@@ -5,25 +5,10 @@
 
 namespace nix {
 
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
-    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
 void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
 {
     Worker worker(*this);
 
-    primeCache(*this, drvPaths);
-
     Goals goals;
     for (auto & path : drvPaths) {
         if (path.path.isDerivation())
@@ -44,9 +29,8 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
                 ex = i->ex;
         }
         if (i->exitCode != Goal::ecSuccess) {
-            DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
-            if (i2) failed.insert(i2->getDrvPath());
-            else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
+            if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
+            else if (auto i2 = dynamic_cast<SubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
         }
     }
 
@@ -84,8 +68,6 @@ void LocalStore::ensurePath(const StorePath & path)
     /* If the path is already valid, we're done. */
     if (isValidPath(path)) return;
 
-    primeCache(*this, {{path}});
-
     Worker worker(*this);
     GoalPtr goal = worker.makeSubstitutionGoal(path);
     Goals goals = {goal};
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index 3ae9a9e6b..dee2cecbf 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -8,11 +8,8 @@ namespace nix {
 
 class Worker;
 
-class SubstitutionGoal : public Goal
+struct SubstitutionGoal : public Goal
 {
-    friend class Worker;
-
-private:
     /* The store path that should be realised through a substitute. */
     StorePath storePath;
 
@@ -56,7 +53,6 @@ private:
     /* Content address for recomputing store path */
     std::optional<ContentAddress> ca;
 
-public:
     SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
     ~SubstitutionGoal();
 
@@ -82,8 +78,6 @@ public:
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
     void handleEOF(int fd) override;
-
-    StorePath getStorePath() { return storePath; }
 };
 
 }
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 17c10cd71..1f8999a4b 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -207,7 +207,26 @@ void Worker::waitForAWhile(GoalPtr goal)
 
 void Worker::run(const Goals & _topGoals)
 {
-    for (auto & i : _topGoals) topGoals.insert(i);
+    std::vector<nix::StorePathWithOutputs> topPaths;
+
+    for (auto & i : _topGoals) {
+        topGoals.insert(i);
+        if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
+            topPaths.push_back({goal->drvPath, goal->wantedOutputs});
+        } else if (auto goal = dynamic_cast<SubstitutionGoal *>(i.get())) {
+            topPaths.push_back({goal->storePath});
+        }
+    }
+
+    /* Call queryMissing() efficiently query substitutes. */
+    StorePathSet willBuild, willSubstitute, unknown;
+    uint64_t downloadSize, narSize;
+    store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
+        throw Error(
+            "%d derivations need to be built, but neither local builds ('--max-jobs') "
+            "nor remote builds ('--builders') are enabled", willBuild.size());
 
     debug("entered goal loop");
 

From c0d1354b7da9ffc2923bc102abb67d03b655fbbf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Nov 2020 12:17:07 +0100
Subject: [PATCH 508/882] Macro hygiene

---
 src/libutil/logging.hh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index ff17056f4..82ba54051 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -191,8 +191,9 @@ extern Verbosity verbosity; /* suppress msgs > this */
    arguments are evaluated lazily. */
 #define printMsg(level, args...) \
     do { \
-        if (level <= nix::verbosity) { \
-            logger->log(level, fmt(args)); \
+        auto __lvl = level; \
+        if (__lvl <= nix::verbosity) { \
+            logger->log(__lvl, fmt(args)); \
         } \
     } while (0)
 

From 88798613ee288c7a801dcc1e73723a10a385df38 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Nov 2020 14:59:03 +0100
Subject: [PATCH 509/882] replaceStrings(): Use std::string_view

---
 src/libutil/util.cc | 6 +++---
 src/libutil/util.hh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 01ab9111f..c1b12e725 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1273,11 +1273,11 @@ string trim(const string & s, const string & whitespace)
 }
 
 
-string replaceStrings(const std::string & s,
+string replaceStrings(std::string_view s,
     const std::string & from, const std::string & to)
 {
-    if (from.empty()) return s;
-    string res = s;
+    string res(s);
+    if (from.empty()) return res;
     size_t pos = 0;
     while ((pos = res.find(from, pos)) != std::string::npos) {
         res.replace(pos, from.size(), to);
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index cafe93702..117fe86e7 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -383,7 +383,7 @@ string trim(const string & s, const string & whitespace = " \n\r\t");
 
 
 /* Replace all occurrences of a string inside another string. */
-string replaceStrings(const std::string & s,
+string replaceStrings(std::string_view s,
     const std::string & from, const std::string & to);
 
 

From 438977731cf049cf47873d5825456d47a1aac541 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 1 Dec 2020 14:57:56 +0100
Subject: [PATCH 510/882] shut up clang warnings

- Fix some class/struct discrepancies
- Explicit the overloading of `run` in the `Cmd*` classes
- Ignore a warning in the generated lexer
---
 src/libexpr/lexer.l          | 2 ++
 src/libstore/build/goal.hh   | 2 +-
 src/libstore/build/worker.hh | 4 ++--
 src/libstore/local-store.hh  | 4 ++--
 src/nix/run.cc               | 8 ++++++++
 5 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index f6e83926b..225ab3287 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -12,6 +12,8 @@
 
 
 %{
+#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
+
 #include <boost/lexical_cast.hpp>
 
 #include "nixexpr.hh"
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 0781a9d38..fca4f2d00 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -7,7 +7,7 @@ namespace nix {
 
 /* Forward definition. */
 struct Goal;
-struct Worker;
+class Worker;
 
 /* A pointer to a goal. */
 typedef std::shared_ptr<Goal> GoalPtr;
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 3a53a8def..bf8cc4586 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -8,8 +8,8 @@
 namespace nix {
 
 /* Forward definition. */
-class DerivationGoal;
-class SubstitutionGoal;
+struct DerivationGoal;
+struct SubstitutionGoal;
 
 /* Workaround for not being able to declare a something like
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index d4435220d..58ec93f27 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -296,8 +296,8 @@ private:
 
     void createUser(const std::string & userName, uid_t userId) override;
 
-    friend class DerivationGoal;
-    friend class SubstitutionGoal;
+    friend struct DerivationGoal;
+    friend struct SubstitutionGoal;
 };
 
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 790784382..92a52c6cd 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -22,6 +22,9 @@ std::string chrootHelperName = "__run_in_chroot";
 
 struct RunCommon : virtual Command
 {
+
+    using Command::run;
+
     void runProgram(ref<Store> store,
         const std::string & program,
         const Strings & args)
@@ -59,6 +62,9 @@ struct RunCommon : virtual Command
 
 struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
 {
+
+    using InstallablesCommand::run;
+
     std::vector<std::string> command = { getEnv("SHELL").value_or("bash") };
 
     CmdShell()
@@ -144,6 +150,8 @@ static auto rCmdShell = registerCommand<CmdShell>("shell");
 
 struct CmdRun : InstallableCommand, RunCommon
 {
+    using InstallableCommand::run;
+
     std::vector<std::string> args;
 
     CmdRun()

From aa684861127eabc18a0d7386a66c5b75d4962897 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 12:43:52 +0100
Subject: [PATCH 511/882] writeFull/writeFile: Use std::string_view

---
 src/libutil/util.cc | 4 ++--
 src/libutil/util.hh | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index c1b12e725..98a069c3e 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -320,7 +320,7 @@ void readFile(const Path & path, Sink & sink)
 }
 
 
-void writeFile(const Path & path, const string & s, mode_t mode)
+void writeFile(const Path & path, std::string_view s, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
@@ -663,7 +663,7 @@ void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterr
 }
 
 
-void writeFull(int fd, const string & s, bool allowInterrupts)
+void writeFull(int fd, std::string_view s, bool allowInterrupts)
 {
     writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts);
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 117fe86e7..eb872dd4b 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -106,7 +106,7 @@ string readFile(const Path & path);
 void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
-void writeFile(const Path & path, const string & s, mode_t mode = 0666);
+void writeFile(const Path & path, std::string_view s, mode_t mode = 0666);
 
 void writeFile(const Path & path, Source & source, mode_t mode = 0666);
 
@@ -157,7 +157,7 @@ void replaceSymlink(const Path & target, const Path & link,
    requested number of bytes. */
 void readFull(int fd, unsigned char * buf, size_t count);
 void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true);
-void writeFull(int fd, const string & s, bool allowInterrupts = true);
+void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
 
 MakeError(EndOfFile, Error);
 

From faa31f40846f7a4dbc2487d000b112a6aef69d1b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 14:00:43 +0100
Subject: [PATCH 512/882] Sink: Use std::string_view

---
 src/libstore/binary-cache-store.cc    |   3 +-
 src/libstore/build/derivation-goal.cc |   8 +-
 src/libstore/daemon.cc                |   4 +-
 src/libstore/filetransfer.cc          |  18 ++--
 src/libstore/filetransfer.hh          |   2 +-
 src/libstore/nar-accessor.cc          |   2 +-
 src/libstore/references.cc            |  50 +++++-------
 src/libstore/references.hh            |   4 +-
 src/libstore/remote-store.cc          |   2 +-
 src/libstore/s3-binary-cache-store.cc |   2 +-
 src/libstore/store-api.cc             |   4 +-
 src/libutil/archive.cc                |  18 ++--
 src/libutil/archive.hh                |   8 +-
 src/libutil/compression.cc            | 113 +++++++++++++-------------
 src/libutil/hash.cc                   |  18 ++--
 src/libutil/hash.hh                   |   2 +-
 src/libutil/serialise.cc              |  48 +++++------
 src/libutil/serialise.hh              |  52 +++++-------
 src/libutil/util.cc                   |  30 +++----
 src/libutil/util.hh                   |   1 -
 src/nix/make-content-addressable.cc   |   4 +-
 21 files changed, 182 insertions(+), 211 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f6224d6a0..a918b7208 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
                 promise.set_exception(std::current_exception());
             }
         }});
-    auto data = promise.get_future().get();
-    sink((unsigned char *) data->data(), data->size());
+    sink(*promise.get_future().get());
 }
 
 std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 76c49f92c..1db85bd37 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -916,10 +916,8 @@ void DerivationGoal::buildDone()
 
                 LogSink(Activity & act) : act(act) { }
 
-                void operator() (const unsigned char * data, size_t len) override {
-                    for (size_t i = 0; i < len; i++) {
-                        auto c = data[i];
-
+                void operator() (std::string_view data) override {
+                    for (auto c : data) {
                         if (c == '\n') {
                             flushLine();
                         } else {
@@ -3127,7 +3125,7 @@ void DerivationGoal::registerOutputs()
                     StringSink sink;
                     dumpPath(actualPath, sink);
                     RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
-                    rsink2((unsigned char *) sink.s->data(), sink.s->size());
+                    rsink2(*sink.s);
                     rsink2.flush();
                 });
                 Path tmpPath = actualPath + ".tmp";
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 60cca4fda..cb214bee3 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -153,10 +153,10 @@ struct TunnelSink : Sink
 {
     Sink & to;
     TunnelSink(Sink & to) : to(to) { }
-    virtual void operator () (const unsigned char * data, size_t len)
+    void operator () (std::string_view data)
     {
         to << STDERR_WRITE;
-        writeString(data, len, to);
+        writeString(data, to);
     }
 };
 
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index c2c65af05..31b4215a9 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
-            , finalSink([this](const unsigned char * data, size_t len) {
+            , finalSink([this](std::string_view data) {
                 if (this->request.dataCallback) {
                     auto httpStatus = getHTTPStatus();
 
                     /* Only write data to the sink if this is a
                        successful response. */
                     if (successfulStatuses.count(httpStatus)) {
-                        writtenToSink += len;
-                        this->request.dataCallback((char *) data, len);
+                        writtenToSink += data.size();
+                        this->request.dataCallback(data);
                     }
                 } else
-                    this->result.data->append((char *) data, len);
+                    this->result.data->append(data);
               })
         {
             if (!request.expectedETag.empty())
@@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer
                 }
 
                 if (errorSink)
-                    (*errorSink)((unsigned char *) contents, realSize);
-                (*decompressionSink)((unsigned char *) contents, realSize);
+                    (*errorSink)({(char *) contents, realSize});
+                (*decompressionSink)({(char *) contents, realSize});
 
                 return realSize;
             } catch (...) {
@@ -776,7 +776,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
         state->request.notify_one();
     });
 
-    request.dataCallback = [_state](char * buf, size_t len) {
+    request.dataCallback = [_state](std::string_view data) {
 
         auto state(_state->lock());
 
@@ -794,7 +794,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
 
         /* Append data to the buffer and wake up the calling
            thread. */
-        state->data.append(buf, len);
+        state->data.append(data);
         state->avail.notify_one();
     };
 
@@ -840,7 +840,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
            if it's blocked on a full buffer. We don't hold the state
            lock while doing this to prevent blocking the download
            thread if sink() takes a long time. */
-        sink((unsigned char *) chunk.data(), chunk.size());
+        sink(chunk);
     }
 }
 
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index c89c51a21..afc7e7aa6 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -61,7 +61,7 @@ struct FileTransferRequest
     bool decompress = true;
     std::shared_ptr<std::string> data;
     std::string mimeType;
-    std::function<void(char *, size_t)> dataCallback;
+    std::function<void(std::string_view data)> dataCallback;
 
     FileTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index a9efdd0b6..e35031f7a 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor
             parents.top()->start = pos;
         }
 
-        void receiveContents(unsigned char * data, size_t len) override
+        void receiveContents(std::string_view data) override
         { }
 
         void createSymlink(const Path & path, const string & target) override
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index d2096cb49..eb117b5ba 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -55,27 +55,23 @@ struct RefScanSink : Sink
 
     RefScanSink() { }
 
-    void operator () (const unsigned char * data, size_t len);
+    void operator () (std::string_view data) override
+    {
+        /* It's possible that a reference spans the previous and current
+           fragment, so search in the concatenation of the tail of the
+           previous fragment and the start of the current fragment. */
+        string s = tail + std::string(data, 0, refLength);
+        search((const unsigned char *) s.data(), s.size(), hashes, seen);
+
+        search((const unsigned char *) data.data(), data.size(), hashes, seen);
+
+        size_t tailLen = data.size() <= refLength ? data.size() : refLength;
+        tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
+        tail.append({data.data() + data.size() - tailLen, tailLen});
+    }
 };
 
 
-void RefScanSink::operator () (const unsigned char * data, size_t len)
-{
-    /* It's possible that a reference spans the previous and current
-       fragment, so search in the concatenation of the tail of the
-       previous fragment and the start of the current fragment. */
-    string s = tail + string((const char *) data, len > refLength ? refLength : len);
-    search((const unsigned char *) s.data(), s.size(), hashes, seen);
-
-    search(data, len, hashes, seen);
-
-    size_t tailLen = len <= refLength ? len : refLength;
-    tail =
-        string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
-        string((const char *) data + len - tailLen, tailLen);
-}
-
-
 std::pair<PathSet, HashResult> scanForReferences(const string & path,
     const PathSet & refs)
 {
@@ -129,10 +125,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S
     assert(from.size() == to.size());
 }
 
-void RewritingSink::operator () (const unsigned char * data, size_t len)
+void RewritingSink::operator () (std::string_view data)
 {
     std::string s(prev);
-    s.append((const char *) data, len);
+    s.append(data);
 
     size_t j = 0;
     while ((j = s.find(from, j)) != string::npos) {
@@ -146,14 +142,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len)
 
     pos += consumed;
 
-    if (consumed) nextSink((unsigned char *) s.data(), consumed);
+    if (consumed) nextSink(s.substr(0, consumed));
 }
 
 void RewritingSink::flush()
 {
     if (prev.empty()) return;
     pos += prev.size();
-    nextSink((unsigned char *) prev.data(), prev.size());
+    nextSink(prev);
     prev.clear();
 }
 
@@ -163,9 +159,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
 {
 }
 
-void HashModuloSink::operator () (const unsigned char * data, size_t len)
+void HashModuloSink::operator () (std::string_view data)
 {
-    rewritingSink(data, len);
+    rewritingSink(data);
 }
 
 HashResult HashModuloSink::finish()
@@ -176,10 +172,8 @@ HashResult HashModuloSink::finish()
        NAR with self-references and a NAR with some of the
        self-references already zeroed out do not produce a hash
        collision. FIXME: proof. */
-    for (auto & pos : rewritingSink.matches) {
-        auto s = fmt("|%d", pos);
-        hashSink((unsigned char *) s.data(), s.size());
-    }
+    for (auto & pos : rewritingSink.matches)
+        hashSink(fmt("|%d", pos));
 
     auto h = hashSink.finish();
     return {h.first, rewritingSink.pos};
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index c2efd095c..4f12e6b21 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -19,7 +19,7 @@ struct RewritingSink : Sink
 
     RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
 
-    void operator () (const unsigned char * data, size_t len) override;
+    void operator () (std::string_view data) override;
 
     void flush();
 };
@@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink
 
     HashModuloSink(HashType ht, const std::string & modulus);
 
-    void operator () (const unsigned char * data, size_t len) override;
+    void operator () (std::string_view data) override;
 
     HashResult finish() override;
 };
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 48af9f0c4..6e0c85237 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -857,7 +857,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
             if (!source) throw Error("no source");
             size_t len = readNum<size_t>(from);
             auto buf = std::make_unique<unsigned char[]>(len);
-            writeString(buf.get(), source->read(buf.get(), len), to);
+            writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
             to.flush();
         }
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 552c4aac7..4519dd5b5 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -398,7 +398,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
             printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
                 bucketName, path, res.data->size(), res.durationMs);
 
-            sink((unsigned char *) res.data->data(), res.data->size());
+            sink(*res.data);
         } else
             throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 3b2be05cb..27be66cac 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -772,8 +772,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     }
 
     auto source = sinkToSource([&](Sink & sink) {
-        LambdaSink progressSink([&](const unsigned char * data, size_t len) {
-            total += len;
+        LambdaSink progressSink([&](std::string_view data) {
+            total += data.size();
             act.progress(total, info->narSize);
         });
         TeeSink tee { sink, progressSink };
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 03534abc4..046ef841b 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -57,7 +57,7 @@ static void dumpContents(const Path & path, size_t size,
         auto n = std::min(left, buf.size());
         readFull(fd.get(), buf.data(), n);
         left -= n;
-        sink(buf.data(), n);
+        sink({(char *) buf.data(), n});
     }
 
     writePadding(size, sink);
@@ -162,7 +162,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
         auto n = buf.size();
         if ((uint64_t)n > left) n = left;
         source(buf.data(), n);
-        sink.receiveContents(buf.data(), n);
+        sink.receiveContents({(char *) buf.data(), n});
         left -= n;
     }
 
@@ -300,21 +300,21 @@ struct RestoreSink : ParseSink
     Path dstPath;
     AutoCloseFD fd;
 
-    void createDirectory(const Path & path)
+    void createDirectory(const Path & path) override
     {
         Path p = dstPath + path;
         if (mkdir(p.c_str(), 0777) == -1)
             throw SysError("creating directory '%1%'", p);
     };
 
-    void createRegularFile(const Path & path)
+    void createRegularFile(const Path & path) override
     {
         Path p = dstPath + path;
         fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
         if (!fd) throw SysError("creating file '%1%'", p);
     }
 
-    void isExecutable()
+    void isExecutable() override
     {
         struct stat st;
         if (fstat(fd.get(), &st) == -1)
@@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
             throw SysError("fchmod");
     }
 
-    void preallocateContents(uint64_t len)
+    void preallocateContents(uint64_t len) override
     {
         if (!archiveSettings.preallocateContents)
             return;
@@ -341,12 +341,12 @@ struct RestoreSink : ParseSink
 #endif
     }
 
-    void receiveContents(unsigned char * data, size_t len)
+    void receiveContents(std::string_view data) override
     {
-        writeFull(fd.get(), data, len);
+        writeFull(fd.get(), data);
     }
 
-    void createSymlink(const Path & path, const string & target)
+    void createSymlink(const Path & path, const string & target) override
     {
         Path p = dstPath + path;
         nix::createSymlink(target, p);
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 5665732d2..fe22435ef 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -58,7 +58,7 @@ struct ParseSink
     virtual void createRegularFile(const Path & path) { };
     virtual void isExecutable() { };
     virtual void preallocateContents(uint64_t size) { };
-    virtual void receiveContents(unsigned char * data, size_t len) { };
+    virtual void receiveContents(std::string_view data) { };
 
     virtual void createSymlink(const Path & path, const string & target) { };
 };
@@ -72,14 +72,14 @@ struct RetrieveRegularNARSink : ParseSink
 
     RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
 
-    void createDirectory(const Path & path)
+    void createDirectory(const Path & path) override
     {
         regular = false;
     }
 
-    void receiveContents(unsigned char * data, size_t len)
+    void receiveContents(std::string_view data) override
     {
-        sink(data, len);
+        sink(data);
     }
 
     void createSymlink(const Path & path, const string & target)
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index a117ddc72..986ba2976 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -22,18 +22,17 @@ struct ChunkedCompressionSink : CompressionSink
 {
     uint8_t outbuf[32 * 1024];
 
-    void write(const unsigned char * data, size_t len) override
+    void write(std::string_view data) override
     {
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
-        while (len) {
-            size_t n = std::min(CHUNK_SIZE, len);
-            writeInternal(data, n);
-            data += n;
-            len -= n;
+        while (!data.empty()) {
+            size_t n = std::min(CHUNK_SIZE, data.size());
+            writeInternal(data);
+            data.remove_prefix(n);
         }
     }
 
-    virtual void writeInternal(const unsigned char * data, size_t len) = 0;
+    virtual void writeInternal(std::string_view data) = 0;
 };
 
 struct NoneSink : CompressionSink
@@ -41,7 +40,7 @@ struct NoneSink : CompressionSink
     Sink & nextSink;
     NoneSink(Sink & nextSink) : nextSink(nextSink) { }
     void finish() override { flush(); }
-    void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
+    void write(std::string_view data) override { nextSink(data); }
 };
 
 struct GzipDecompressionSink : CompressionSink
@@ -75,28 +74,28 @@ struct GzipDecompressionSink : CompressionSink
     void finish() override
     {
         CompressionSink::flush();
-        write(nullptr, 0);
+        write({});
     }
 
-    void write(const unsigned char * data, size_t len) override
+    void write(std::string_view data) override
     {
-        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
-        strm.next_in = (Bytef *) data;
-        strm.avail_in = len;
+        strm.next_in = (Bytef *) data.data();
+        strm.avail_in = data.size();
 
-        while (!finished && (!data || strm.avail_in)) {
+        while (!finished && (!data.data() || strm.avail_in)) {
             checkInterrupt();
 
             int ret = inflate(&strm,Z_SYNC_FLUSH);
             if (ret != Z_OK && ret != Z_STREAM_END)
                 throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
-                    zError(ret), len, strm.avail_in);
+                    zError(ret), data.size(), strm.avail_in);
 
             finished = ret == Z_STREAM_END;
 
             if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
                 strm.next_out = (Bytef *) outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -130,25 +129,25 @@ struct XzDecompressionSink : CompressionSink
     void finish() override
     {
         CompressionSink::flush();
-        write(nullptr, 0);
+        write({});
     }
 
-    void write(const unsigned char * data, size_t len) override
+    void write(std::string_view data) override
     {
-        strm.next_in = data;
-        strm.avail_in = len;
+        strm.next_in = (const unsigned char *) data.data();
+        strm.avail_in = data.size();
 
-        while (!finished && (!data || strm.avail_in)) {
+        while (!finished && (!data.data() || strm.avail_in)) {
             checkInterrupt();
 
-            lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+            lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
             if (ret != LZMA_OK && ret != LZMA_STREAM_END)
                 throw CompressionError("error %d while decompressing xz file", ret);
 
             finished = ret == LZMA_STREAM_END;
 
             if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
                 strm.next_out = outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -181,15 +180,15 @@ struct BzipDecompressionSink : ChunkedCompressionSink
     void finish() override
     {
         flush();
-        write(nullptr, 0);
+        write({});
     }
 
-    void writeInternal(const unsigned char * data, size_t len) override
+    void writeInternal(std::string_view data) override
     {
-        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
-        strm.next_in = (char *) data;
-        strm.avail_in = len;
+        strm.next_in = (char *) data.data();
+        strm.avail_in = data.size();
 
         while (strm.avail_in) {
             checkInterrupt();
@@ -201,7 +200,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink
             finished = ret == BZ_STREAM_END;
 
             if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
                 strm.next_out = (char *) outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -230,17 +229,17 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
     void finish() override
     {
         flush();
-        writeInternal(nullptr, 0);
+        writeInternal({});
     }
 
-    void writeInternal(const unsigned char * data, size_t len) override
+    void writeInternal(std::string_view data) override
     {
-        const uint8_t * next_in = data;
-        size_t avail_in = len;
+        auto next_in = (const uint8_t *) data.data();
+        size_t avail_in = data.size();
         uint8_t * next_out = outbuf;
         size_t avail_out = sizeof(outbuf);
 
-        while (!finished && (!data || avail_in)) {
+        while (!finished && (!data.data() || avail_in)) {
             checkInterrupt();
 
             if (!BrotliDecoderDecompressStream(state,
@@ -250,7 +249,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
                 throw CompressionError("error while decompressing brotli file");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                nextSink({(char *) outbuf, sizeof(outbuf) - avail_out});
                 next_out = outbuf;
                 avail_out = sizeof(outbuf);
             }
@@ -338,25 +337,25 @@ struct XzCompressionSink : CompressionSink
     void finish() override
     {
         CompressionSink::flush();
-        write(nullptr, 0);
+        write({});
     }
 
-    void write(const unsigned char * data, size_t len) override
+    void write(std::string_view data) override
     {
-        strm.next_in = data;
-        strm.avail_in = len;
+        strm.next_in = (const unsigned char *) data.data();
+        strm.avail_in = data.size();
 
-        while (!finished && (!data || strm.avail_in)) {
+        while (!finished && (!data.data() || strm.avail_in)) {
             checkInterrupt();
 
-            lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+            lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
             if (ret != LZMA_OK && ret != LZMA_STREAM_END)
                 throw CompressionError("error %d while compressing xz file", ret);
 
             finished = ret == LZMA_STREAM_END;
 
             if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
                 strm.next_out = outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -389,27 +388,27 @@ struct BzipCompressionSink : ChunkedCompressionSink
     void finish() override
     {
         flush();
-        writeInternal(nullptr, 0);
+        writeInternal({});
     }
 
-    void writeInternal(const unsigned char * data, size_t len) override
+    void writeInternal(std::string_view data) override
     {
-        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
-        strm.next_in = (char *) data;
-        strm.avail_in = len;
+        strm.next_in = (char *) data.data();
+        strm.avail_in = data.size();
 
-        while (!finished && (!data || strm.avail_in)) {
+        while (!finished && (!data.data() || strm.avail_in)) {
             checkInterrupt();
 
-            int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
+            int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
             if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
                 throw CompressionError("error %d while compressing bzip2 file", ret);
 
             finished = ret == BZ_STREAM_END;
 
             if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
                 strm.next_out = (char *) outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -439,28 +438,28 @@ struct BrotliCompressionSink : ChunkedCompressionSink
     void finish() override
     {
         flush();
-        writeInternal(nullptr, 0);
+        writeInternal({});
     }
 
-    void writeInternal(const unsigned char * data, size_t len) override
+    void writeInternal(std::string_view data) override
     {
-        const uint8_t * next_in = data;
-        size_t avail_in = len;
+        auto next_in = (const uint8_t *) data.data();
+        size_t avail_in = data.size();
         uint8_t * next_out = outbuf;
         size_t avail_out = sizeof(outbuf);
 
-        while (!finished && (!data || avail_in)) {
+        while (!finished && (!data.data() || avail_in)) {
             checkInterrupt();
 
             if (!BrotliEncoderCompressStream(state,
-                    data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
+                    data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
                     &avail_in, &next_in,
                     &avail_out, &next_out,
                     nullptr))
                 throw CompressionError("error while compressing brotli compression");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
-                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                nextSink({(const char *) outbuf, sizeof(outbuf) - avail_out});
                 next_out = outbuf;
                 avail_out = sizeof(outbuf);
             }
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 8efff190a..4df8b4ecb 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -291,12 +291,12 @@ static void start(HashType ht, Ctx & ctx)
 
 
 static void update(HashType ht, Ctx & ctx,
-    const unsigned char * bytes, size_t len)
+    std::string_view data)
 {
-    if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len);
-    else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len);
-    else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len);
-    else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len);
+    if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
+    else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
+    else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
 }
 
 
@@ -314,7 +314,7 @@ Hash hashString(HashType ht, std::string_view s)
     Ctx ctx;
     Hash hash(ht);
     start(ht, ctx);
-    update(ht, ctx, (const unsigned char *) s.data(), s.length());
+    update(ht, ctx, s);
     finish(ht, ctx, hash.hash);
     return hash;
 }
@@ -341,10 +341,10 @@ HashSink::~HashSink()
     delete ctx;
 }
 
-void HashSink::write(const unsigned char * data, size_t len)
+void HashSink::write(std::string_view data)
 {
-    bytes += len;
-    update(ht, *ctx, data, len);
+    bytes += data.size();
+    update(ht, *ctx, data);
 }
 
 HashResult HashSink::finish()
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 6d6eb70ca..1b626dd85 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -156,7 +156,7 @@ public:
     HashSink(HashType ht);
     HashSink(const HashSink & h);
     ~HashSink();
-    void write(const unsigned char * data, size_t len) override;
+    void write(std::string_view data) override;
     HashResult finish() override;
     HashResult currentHash();
 };
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 038ede049..534ad5cbb 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -11,23 +11,23 @@
 namespace nix {
 
 
-void BufferedSink::operator () (const unsigned char * data, size_t len)
+void BufferedSink::operator () (std::string_view data)
 {
-    if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
+    if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
 
-    while (len) {
+    while (!data.empty()) {
         /* Optimisation: bypass the buffer if the data exceeds the
            buffer size. */
-        if (bufPos + len >= bufSize) {
+        if (bufPos + data.size() >= bufSize) {
             flush();
-            write(data, len);
+            write(data);
             break;
         }
         /* Otherwise, copy the bytes to the buffer.  Flush the buffer
            when it's full. */
-        size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
-        memcpy(buffer.get() + bufPos, data, n);
-        data += n; bufPos += n; len -= n;
+        size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size();
+        memcpy(buffer.get() + bufPos, data.data(), n);
+        data.remove_prefix(n); bufPos += n;
         if (bufPos == bufSize) flush();
     }
 }
@@ -38,7 +38,7 @@ void BufferedSink::flush()
     if (bufPos == 0) return;
     size_t n = bufPos;
     bufPos = 0; // don't trigger the assert() in ~BufferedSink()
-    write(buffer.get(), n);
+    write({buffer.get(), n});
 }
 
 
@@ -59,9 +59,9 @@ static void warnLargeDump()
 }
 
 
-void FdSink::write(const unsigned char * data, size_t len)
+void FdSink::write(std::string_view data)
 {
-    written += len;
+    written += data.size();
     static bool warned = false;
     if (warn && !warned) {
         if (written > threshold) {
@@ -70,7 +70,7 @@ void FdSink::write(const unsigned char * data, size_t len)
         }
     }
     try {
-        writeFull(fd, data, len);
+        writeFull(fd, data);
     } catch (SysError & e) {
         _good = false;
         throw;
@@ -101,7 +101,7 @@ void Source::drainInto(Sink & sink)
         size_t n;
         try {
             n = read(buf.data(), buf.size());
-            sink(buf.data(), n);
+            sink({(char *) buf.data(), n});
         } catch (EndOfFile &) {
             break;
         }
@@ -229,9 +229,9 @@ std::unique_ptr<Source> sinkToSource(
         {
             if (!coro)
                 coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
-                    LambdaSink sink([&](const unsigned char * data, size_t len) {
-                            if (len) yield(std::string((const char *) data, len));
-                        });
+                    LambdaSink sink([&](std::string_view data) {
+                        if (!data.empty()) yield(std::string(data));
+                    });
                     fun(sink);
                 });
 
@@ -260,22 +260,22 @@ void writePadding(size_t len, Sink & sink)
     if (len % 8) {
         unsigned char zero[8];
         memset(zero, 0, sizeof(zero));
-        sink(zero, 8 - (len % 8));
+        sink({(char *) zero, 8 - (len % 8)});
     }
 }
 
 
-void writeString(const unsigned char * buf, size_t len, Sink & sink)
+void writeString(std::string_view data, Sink & sink)
 {
-    sink << len;
-    sink(buf, len);
-    writePadding(len, sink);
+    sink << data.size();
+    sink(data);
+    writePadding(data.size(), sink);
 }
 
 
 Sink & operator << (Sink & sink, const string & s)
 {
-    writeString((const unsigned char *) s.data(), s.size(), sink);
+    writeString(s, sink);
     return sink;
 }
 
@@ -394,14 +394,14 @@ Error readError(Source & source)
 }
 
 
-void StringSink::operator () (const unsigned char * data, size_t len)
+void StringSink::operator () (std::string_view data)
 {
     static bool warned = false;
     if (!warned && s->size() > threshold) {
         warnLargeDump();
         warned = true;
     }
-    s->append((const char *) data, len);
+    s->append(data);
 }
 
 size_t ChainSource::read(unsigned char * data, size_t len)
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5c7d3ce76..f9ab79997 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -14,19 +14,14 @@ namespace nix {
 struct Sink
 {
     virtual ~Sink() { }
-    virtual void operator () (const unsigned char * data, size_t len) = 0;
+    virtual void operator () (std::string_view data) = 0;
     virtual bool good() { return true; }
-
-    void operator () (const std::string & s)
-    {
-        (*this)((const unsigned char *) s.data(), s.size());
-    }
 };
 
 /* Just throws away data. */
 struct NullSink : Sink
 {
-    void operator () (const unsigned char * data, size_t len) override
+    void operator () (std::string_view data) override
     { }
 };
 
@@ -35,21 +30,16 @@ struct NullSink : Sink
 struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
-    std::unique_ptr<unsigned char[]> buffer;
+    std::unique_ptr<char[]> buffer;
 
     BufferedSink(size_t bufSize = 32 * 1024)
         : bufSize(bufSize), bufPos(0), buffer(nullptr) { }
 
-    void operator () (const unsigned char * data, size_t len) override;
-
-    void operator () (const std::string & s)
-    {
-        Sink::operator()(s);
-    }
+    void operator () (std::string_view data) override;
 
     void flush();
 
-    virtual void write(const unsigned char * data, size_t len) = 0;
+    virtual void write(std::string_view data) = 0;
 };
 
 
@@ -119,7 +109,7 @@ struct FdSink : BufferedSink
 
     ~FdSink();
 
-    void write(const unsigned char * data, size_t len) override;
+    void write(std::string_view data) override;
 
     bool good() override;
 
@@ -163,7 +153,7 @@ struct StringSink : Sink
       s->reserve(reservedSize);
     };
     StringSink(ref<std::string> s) : s(s) { };
-    void operator () (const unsigned char * data, size_t len) override;
+    void operator () (std::string_view data) override;
 };
 
 
@@ -182,10 +172,10 @@ struct TeeSink : Sink
 {
     Sink & sink1, & sink2;
     TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
-    virtual void operator () (const unsigned char * data, size_t len)
+    virtual void operator () (std::string_view data)
     {
-        sink1(data, len);
-        sink2(data, len);
+        sink1(data);
+        sink2(data);
     }
 };
 
@@ -200,7 +190,7 @@ struct TeeSource : Source
     size_t read(unsigned char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        sink(data, n);
+        sink({(char *) data, n});
         return n;
     }
 };
@@ -241,24 +231,24 @@ struct LengthSink : Sink
 {
     uint64_t length = 0;
 
-    virtual void operator () (const unsigned char * _, size_t len)
+    void operator () (std::string_view data) override
     {
-        length += len;
+        length += data.size();
     }
 };
 
 /* Convert a function into a sink. */
 struct LambdaSink : Sink
 {
-    typedef std::function<void(const unsigned char *, size_t)> lambda_t;
+    typedef std::function<void(std::string_view data)> lambda_t;
 
     lambda_t lambda;
 
     LambdaSink(const lambda_t & lambda) : lambda(lambda) { }
 
-    virtual void operator () (const unsigned char * data, size_t len)
+    void operator () (std::string_view data) override
     {
-        lambda(data, len);
+        lambda(data);
     }
 };
 
@@ -302,7 +292,7 @@ std::unique_ptr<Source> sinkToSource(
 
 
 void writePadding(size_t len, Sink & sink);
-void writeString(const unsigned char * buf, size_t len, Sink & sink);
+void writeString(std::string_view s, Sink & sink);
 
 inline Sink & operator << (Sink & sink, uint64_t n)
 {
@@ -315,7 +305,7 @@ inline Sink & operator << (Sink & sink, uint64_t n)
     buf[5] = (n >> 40) & 0xff;
     buf[6] = (n >> 48) & 0xff;
     buf[7] = (unsigned char) (n >> 56) & 0xff;
-    sink(buf, sizeof(buf));
+    sink({(char *) buf, sizeof(buf)});
     return sink;
 }
 
@@ -484,7 +474,7 @@ struct FramedSink : nix::BufferedSink
         }
     }
 
-    void write(const unsigned char * data, size_t len) override
+    void write(std::string_view data) override
     {
         /* Don't send more data if the remote has
             encountered an error. */
@@ -493,8 +483,8 @@ struct FramedSink : nix::BufferedSink
             ex = nullptr;
             std::rethrow_exception(ex2);
         }
-        to << len;
-        to(data, len);
+        to << data.size();
+        to(data);
     };
 };
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 98a069c3e..3d4fc4c25 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -346,7 +346,7 @@ void writeFile(const Path & path, Source & source, mode_t mode)
         while (true) {
             try {
                 auto n = source.read(buf.data(), buf.size());
-                writeFull(fd.get(), (unsigned char *) buf.data(), n);
+                writeFull(fd.get(), {(char *) buf.data(), n});
             } catch (EndOfFile &) { break; }
         }
     } catch (Error & e) {
@@ -648,24 +648,16 @@ void readFull(int fd, unsigned char * buf, size_t count)
 }
 
 
-void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts)
-{
-    while (count) {
-        if (allowInterrupts) checkInterrupt();
-        ssize_t res = write(fd, (char *) buf, count);
-        if (res == -1 && errno != EINTR)
-            throw SysError("writing to file");
-        if (res > 0) {
-            count -= res;
-            buf += res;
-        }
-    }
-}
-
-
 void writeFull(int fd, std::string_view s, bool allowInterrupts)
 {
-    writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts);
+    while (!s.empty()) {
+        if (allowInterrupts) checkInterrupt();
+        ssize_t res = write(fd, s.data(), s.size());
+        if (res == -1 && errno != EINTR)
+            throw SysError("writing to file");
+        if (res > 0)
+            s.remove_prefix(res);
+    }
 }
 
 
@@ -705,7 +697,7 @@ void drainFD(int fd, Sink & sink, bool block)
                 throw SysError("reading from file");
         }
         else if (rd == 0) break;
-        else sink(buf.data(), rd);
+        else sink({(char *) buf.data(), (size_t) rd});
     }
 }
 
@@ -1153,7 +1145,7 @@ void runProgram2(const RunOptions & options)
                     } catch (EndOfFile &) {
                         break;
                     }
-                    writeFull(in.writeSide.get(), buf.data(), n);
+                    writeFull(in.writeSide.get(), {(char *) buf.data(), n});
                 }
                 promise.set_value();
             } catch (...) {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index eb872dd4b..fcb7cad39 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -156,7 +156,6 @@ void replaceSymlink(const Path & target, const Path & link,
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */
 void readFull(int fd, unsigned char * buf, size_t count);
-void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true);
 void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
 
 MakeError(EndOfFile, Error);
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index df3ec5194..12c2cf776 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -73,7 +73,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             *sink.s = rewriteStrings(*sink.s, rewrites);
 
             HashModuloSink hashModuloSink(htSHA256, oldHashPart);
-            hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
+            hashModuloSink(*sink.s);
 
             auto narHash = hashModuloSink.finish().first;
 
@@ -94,7 +94,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
             auto source = sinkToSource([&](Sink & nextSink) {
                 RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
-                rsink2((unsigned char *) sink.s->data(), sink.s->size());
+                rsink2(*sink.s);
                 rsink2.flush();
             });
 

From 1b79b5b983a6c775766bd0d1c7881042188998b8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 14:10:56 +0100
Subject: [PATCH 513/882] read(): Use char * instead of unsigned char *

This gets rid of some pointless casts.
---
 src/libstore/daemon.cc             |  2 +-
 src/libstore/local-store.cc        |  2 +-
 src/libstore/nar-accessor.cc       |  2 +-
 src/libstore/remote-fs-accessor.cc |  2 +-
 src/libstore/remote-store.cc       |  2 +-
 src/libutil/archive.cc             |  8 +++---
 src/libutil/serialise.cc           | 34 +++++++++++------------
 src/libutil/serialise.hh           | 44 +++++++++++++++---------------
 src/libutil/tarfile.cc             |  2 +-
 src/libutil/util.cc                | 12 ++++----
 src/libutil/util.hh                |  2 +-
 11 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index cb214bee3..e5cfe94cb 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -165,7 +165,7 @@ struct TunnelSource : BufferedSource
     Source & from;
     BufferedSink & to;
     TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { }
-    size_t readUnbuffered(unsigned char * data, size_t len) override
+    size_t readUnbuffered(char * data, size_t len) override
     {
         to << STDERR_READ << len;
         to.flush();
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 93d073768..348e5d0d4 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1143,7 +1143,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
         dump.resize(oldSize + want);
         auto got = 0;
         try {
-            got = source.read((uint8_t *) dump.data() + oldSize, want);
+            got = source.read(dump.data() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index e35031f7a..1427a0f98 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -96,7 +96,7 @@ struct NarAccessor : public FSAccessor
                 NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
         }
 
-        size_t read(unsigned char * data, size_t len) override
+        size_t read(char * data, size_t len) override
         {
             auto n = source.read(data, len);
             pos += n;
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 2d02a181b..63bde92de 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -75,7 +75,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
                         throw SysError("seeking in '%s'", cacheFile);
 
                     std::string buf(length, 0);
-                    readFull(fd.get(), (unsigned char *) buf.data(), length);
+                    readFull(fd.get(), buf.data(), length);
 
                     return buf;
                 });
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 6e0c85237..be29f8e6f 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -856,7 +856,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         else if (msg == STDERR_READ) {
             if (!source) throw Error("no source");
             size_t len = readNum<size_t>(from);
-            auto buf = std::make_unique<unsigned char[]>(len);
+            auto buf = std::make_unique<char[]>(len);
             writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
             to.flush();
         }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 046ef841b..ed0eb2fb5 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -50,14 +50,14 @@ static void dumpContents(const Path & path, size_t size,
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd) throw SysError("opening file '%1%'", path);
 
-    std::vector<unsigned char> buf(65536);
+    std::vector<char> buf(65536);
     size_t left = size;
 
     while (left > 0) {
         auto n = std::min(left, buf.size());
         readFull(fd.get(), buf.data(), n);
         left -= n;
-        sink({(char *) buf.data(), n});
+        sink({buf.data(), n});
     }
 
     writePadding(size, sink);
@@ -155,14 +155,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
     sink.preallocateContents(size);
 
     uint64_t left = size;
-    std::vector<unsigned char> buf(65536);
+    std::vector<char> buf(65536);
 
     while (left) {
         checkInterrupt();
         auto n = buf.size();
         if ((uint64_t)n > left) n = left;
         source(buf.data(), n);
-        sink.receiveContents({(char *) buf.data(), n});
+        sink.receiveContents({buf.data(), n});
         left -= n;
     }
 
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 534ad5cbb..87c1099a1 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -84,7 +84,7 @@ bool FdSink::good()
 }
 
 
-void Source::operator () (unsigned char * data, size_t len)
+void Source::operator () (char * data, size_t len)
 {
     while (len) {
         size_t n = read(data, len);
@@ -96,12 +96,12 @@ void Source::operator () (unsigned char * data, size_t len)
 void Source::drainInto(Sink & sink)
 {
     std::string s;
-    std::vector<unsigned char> buf(8192);
+    std::vector<char> buf(8192);
     while (true) {
         size_t n;
         try {
             n = read(buf.data(), buf.size());
-            sink({(char *) buf.data(), n});
+            sink({buf.data(), n});
         } catch (EndOfFile &) {
             break;
         }
@@ -117,9 +117,9 @@ std::string Source::drain()
 }
 
 
-size_t BufferedSource::read(unsigned char * data, size_t len)
+size_t BufferedSource::read(char * data, size_t len)
 {
-    if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
+    if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
 
     if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize);
 
@@ -138,12 +138,12 @@ bool BufferedSource::hasData()
 }
 
 
-size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
+size_t FdSource::readUnbuffered(char * data, size_t len)
 {
     ssize_t n;
     do {
         checkInterrupt();
-        n = ::read(fd, (char *) data, len);
+        n = ::read(fd, data, len);
     } while (n == -1 && errno == EINTR);
     if (n == -1) { _good = false; throw SysError("reading from file"); }
     if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
@@ -158,10 +158,10 @@ bool FdSource::good()
 }
 
 
-size_t StringSource::read(unsigned char * data, size_t len)
+size_t StringSource::read(char * data, size_t len)
 {
     if (pos == s.size()) throw EndOfFile("end of string reached");
-    size_t n = s.copy((char *) data, len, pos);
+    size_t n = s.copy(data, len, pos);
     pos += n;
     return n;
 }
@@ -225,7 +225,7 @@ std::unique_ptr<Source> sinkToSource(
         std::string cur;
         size_t pos = 0;
 
-        size_t read(unsigned char * data, size_t len) override
+        size_t read(char * data, size_t len) override
         {
             if (!coro)
                 coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
@@ -244,7 +244,7 @@ std::unique_ptr<Source> sinkToSource(
             }
 
             auto n = std::min(cur.size() - pos, len);
-            memcpy(data, (unsigned char *) cur.data() + pos, n);
+            memcpy(data, cur.data() + pos, n);
             pos += n;
 
             return n;
@@ -258,9 +258,9 @@ std::unique_ptr<Source> sinkToSource(
 void writePadding(size_t len, Sink & sink)
 {
     if (len % 8) {
-        unsigned char zero[8];
+        char zero[8];
         memset(zero, 0, sizeof(zero));
-        sink({(char *) zero, 8 - (len % 8)});
+        sink({zero, 8 - (len % 8)});
     }
 }
 
@@ -321,7 +321,7 @@ Sink & operator << (Sink & sink, const Error & ex)
 void readPadding(size_t len, Source & source)
 {
     if (len % 8) {
-        unsigned char zero[8];
+        char zero[8];
         size_t n = 8 - (len % 8);
         source(zero, n);
         for (unsigned int i = 0; i < n; i++)
@@ -330,7 +330,7 @@ void readPadding(size_t len, Source & source)
 }
 
 
-size_t readString(unsigned char * buf, size_t max, Source & source)
+size_t readString(char * buf, size_t max, Source & source)
 {
     auto len = readNum<size_t>(source);
     if (len > max) throw SerialisationError("string is too long");
@@ -345,7 +345,7 @@ string readString(Source & source, size_t max)
     auto len = readNum<size_t>(source);
     if (len > max) throw SerialisationError("string is too long");
     std::string res(len, 0);
-    source((unsigned char*) res.data(), len);
+    source(res.data(), len);
     readPadding(len, source);
     return res;
 }
@@ -404,7 +404,7 @@ void StringSink::operator () (std::string_view data)
     s->append(data);
 }
 
-size_t ChainSource::read(unsigned char * data, size_t len)
+size_t ChainSource::read(char * data, size_t len)
 {
     if (useSecond) {
         return source2.read(data, len);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index f9ab79997..5bbbc7ce3 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -51,12 +51,12 @@ struct Source
     /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’.
        It blocks until all the requested data is available, or throws
        an error if it is not going to be available.   */
-    void operator () (unsigned char * data, size_t len);
+    void operator () (char * data, size_t len);
 
     /* Store up to ‘len’ in the buffer pointed to by ‘data’, and
        return the number of bytes stored.  It blocks until at least
        one byte is available. */
-    virtual size_t read(unsigned char * data, size_t len) = 0;
+    virtual size_t read(char * data, size_t len) = 0;
 
     virtual bool good() { return true; }
 
@@ -71,18 +71,18 @@ struct Source
 struct BufferedSource : Source
 {
     size_t bufSize, bufPosIn, bufPosOut;
-    std::unique_ptr<unsigned char[]> buffer;
+    std::unique_ptr<char[]> buffer;
 
     BufferedSource(size_t bufSize = 32 * 1024)
         : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { }
 
-    size_t read(unsigned char * data, size_t len) override;
+    size_t read(char * data, size_t len) override;
 
     bool hasData();
 
 protected:
     /* Underlying read call, to be overridden. */
-    virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
+    virtual size_t readUnbuffered(char * data, size_t len) = 0;
 };
 
 
@@ -138,7 +138,7 @@ struct FdSource : BufferedSource
 
     bool good() override;
 protected:
-    size_t readUnbuffered(unsigned char * data, size_t len) override;
+    size_t readUnbuffered(char * data, size_t len) override;
 private:
     bool _good = true;
 };
@@ -163,7 +163,7 @@ struct StringSource : Source
     const string & s;
     size_t pos;
     StringSource(const string & _s) : s(_s), pos(0) { }
-    size_t read(unsigned char * data, size_t len) override;
+    size_t read(char * data, size_t len) override;
 };
 
 
@@ -187,10 +187,10 @@ struct TeeSource : Source
     Sink & sink;
     TeeSource(Source & orig, Sink & sink)
         : orig(orig), sink(sink) { }
-    size_t read(unsigned char * data, size_t len)
+    size_t read(char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        sink({(char *) data, n});
+        sink({data, n});
         return n;
     }
 };
@@ -202,7 +202,7 @@ struct SizedSource : Source
     size_t remain;
     SizedSource(Source & orig, size_t size)
         : orig(orig), remain(size) { }
-    size_t read(unsigned char * data, size_t len)
+    size_t read(char * data, size_t len)
     {
         if (this->remain <= 0) {
             throw EndOfFile("sized: unexpected end-of-file");
@@ -216,7 +216,7 @@ struct SizedSource : Source
     /* Consume the original source until no remain data is left to consume. */
     size_t drainAll()
     {
-        std::vector<unsigned char> buf(8192);
+        std::vector<char> buf(8192);
         size_t sum = 0;
         while (this->remain > 0) {
             size_t n = read(buf.data(), buf.size());
@@ -256,13 +256,13 @@ struct LambdaSink : Sink
 /* Convert a function into a source. */
 struct LambdaSource : Source
 {
-    typedef std::function<size_t(unsigned char *, size_t)> lambda_t;
+    typedef std::function<size_t(char *, size_t)> lambda_t;
 
     lambda_t lambda;
 
     LambdaSource(const lambda_t & lambda) : lambda(lambda) { }
 
-    size_t read(unsigned char * data, size_t len) override
+    size_t read(char * data, size_t len) override
     {
         return lambda(data, len);
     }
@@ -278,7 +278,7 @@ struct ChainSource : Source
         : source1(s1), source2(s2)
     { }
 
-    size_t read(unsigned char * data, size_t len) override;
+    size_t read(char * data, size_t len) override;
 };
 
 
@@ -322,7 +322,7 @@ template<typename T>
 T readNum(Source & source)
 {
     unsigned char buf[8];
-    source(buf, sizeof(buf));
+    source((char *) buf, sizeof(buf));
 
     uint64_t n =
         ((uint64_t) buf[0]) |
@@ -354,7 +354,7 @@ inline uint64_t readLongLong(Source & source)
 
 
 void readPadding(size_t len, Source & source);
-size_t readString(unsigned char * buf, size_t max, Source & source);
+size_t readString(char * buf, size_t max, Source & source);
 string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
 template<class T> T readStrings(Source & source);
 
@@ -386,9 +386,9 @@ struct StreamToSourceAdapter : Source
         : istream(istream)
     { }
 
-    size_t read(unsigned char * data, size_t len) override
+    size_t read(char * data, size_t len) override
     {
-        if (!istream->read((char *) data, len)) {
+        if (!istream->read(data, len)) {
             if (istream->eof()) {
                 if (istream->gcount() == 0)
                     throw EndOfFile("end of file");
@@ -411,7 +411,7 @@ struct FramedSource : Source
 {
     Source & from;
     bool eof = false;
-    std::vector<unsigned char> pending;
+    std::vector<char> pending;
     size_t pos = 0;
 
     FramedSource(Source & from) : from(from)
@@ -423,13 +423,13 @@ struct FramedSource : Source
             while (true) {
                 auto n = readInt(from);
                 if (!n) break;
-                std::vector<unsigned char> data(n);
+                std::vector<char> data(n);
                 from(data.data(), n);
             }
         }
     }
 
-    size_t read(unsigned char * data, size_t len) override
+    size_t read(char * data, size_t len) override
     {
         if (eof) throw EndOfFile("reached end of FramedSource");
 
@@ -439,7 +439,7 @@ struct FramedSource : Source
                 eof = true;
                 return 0;
             }
-            pending = std::vector<unsigned char>(len);
+            pending = std::vector<char>(len);
             pos = 0;
             from(pending.data(), len);
         }
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index c4d8a4f91..2da169ba7 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -66,7 +66,7 @@ private:
         *buffer = self->buffer.data();
 
         try {
-            return self->source->read(self->buffer.data(), 4096);
+            return self->source->read((char *) self->buffer.data(), 4096);
         } catch (EndOfFile &) {
             return 0;
         } catch (std::exception & err) {
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 3d4fc4c25..e6b6d287d 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -340,13 +340,13 @@ void writeFile(const Path & path, Source & source, mode_t mode)
     if (!fd)
         throw SysError("opening file '%1%'", path);
 
-    std::vector<unsigned char> buf(64 * 1024);
+    std::vector<char> buf(64 * 1024);
 
     try {
         while (true) {
             try {
                 auto n = source.read(buf.data(), buf.size());
-                writeFull(fd.get(), {(char *) buf.data(), n});
+                writeFull(fd.get(), {buf.data(), n});
             } catch (EndOfFile &) { break; }
         }
     } catch (Error & e) {
@@ -632,11 +632,11 @@ void replaceSymlink(const Path & target, const Path & link,
 }
 
 
-void readFull(int fd, unsigned char * buf, size_t count)
+void readFull(int fd, char * buf, size_t count)
 {
     while (count) {
         checkInterrupt();
-        ssize_t res = read(fd, (char *) buf, count);
+        ssize_t res = read(fd, buf, count);
         if (res == -1) {
             if (errno == EINTR) continue;
             throw SysError("reading from file");
@@ -1137,7 +1137,7 @@ void runProgram2(const RunOptions & options)
         in.readSide = -1;
         writerThread = std::thread([&]() {
             try {
-                std::vector<unsigned char> buf(8 * 1024);
+                std::vector<char> buf(8 * 1024);
                 while (true) {
                     size_t n;
                     try {
@@ -1145,7 +1145,7 @@ void runProgram2(const RunOptions & options)
                     } catch (EndOfFile &) {
                         break;
                     }
-                    writeFull(in.writeSide.get(), {(char *) buf.data(), n});
+                    writeFull(in.writeSide.get(), {buf.data(), n});
                 }
                 promise.set_value();
             } catch (...) {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index fcb7cad39..0f82bed78 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -155,7 +155,7 @@ void replaceSymlink(const Path & target, const Path & link,
 
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */
-void readFull(int fd, unsigned char * buf, size_t count);
+void readFull(int fd, char * buf, size_t count);
 void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
 
 MakeError(EndOfFile, Error);

From a8a96dbaf8ae4f49491903876ddaba44be9d3a39 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 2 Dec 2020 14:23:38 +0100
Subject: [PATCH 514/882] Add forgotten `override` annotation

---
 src/libutil/archive.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index fe22435ef..9e9e11b1a 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -82,7 +82,7 @@ struct RetrieveRegularNARSink : ParseSink
         sink(data);
     }
 
-    void createSymlink(const Path & path, const string & target)
+    void createSymlink(const Path & path, const string & target) override
     {
         regular = false;
     }

From 0d9e1af695ac238b0f8c8bc296bf01eb1430e26b Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 2 Dec 2020 14:33:20 +0100
Subject: [PATCH 515/882] Remove an `unknown pragma` gcc warning

---
 src/libexpr/lexer.l | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 225ab3287..7298419d9 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -12,7 +12,9 @@
 
 
 %{
+#ifdef __clang__
 #pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
+#endif
 
 #include <boost/lexical_cast.hpp>
 

From d8fc1bb7b00dd7b13d667d3cb41bfcbe0df699d0 Mon Sep 17 00:00:00 2001
From: Greg Hale <ghale@octoml.ai>
Date: Wed, 2 Dec 2020 10:15:18 -0500
Subject: [PATCH 516/882] fix tokens documentation

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index eabd83e3f..4655ca058 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -867,7 +867,7 @@ public:
           Example `~/.config/nix/nix.conf`:
 
           ```
-          access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+          access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk
           ```
 
           Example `~/code/flake.nix`:

From 44da19f73cfa87bb09c14fe64fce97904db0970c Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Wed, 2 Dec 2020 17:00:32 +0100
Subject: [PATCH 517/882] Adds Nix CLI Guideline to docs

As we are working towards Nix 3.0 we want to make sure that we make a
huge step forward in Nix's user experience. And once 3.0 is out of the
door we need to make sure that all future commands and features keep up
the standard of user experience.

This PR adds a CLI guideline document to the Nix documentation. Consider
this document a good starting point and a checklist when somebody will
be (re)implementing commands.

Clearly this guideline does nothing to improve user experience on its
own and can only be useful as long as it is going to be read and
cared for. But it is a first step into that direction.
---
 doc/manual/src/SUMMARY.md                   |   1 +
 doc/manual/src/command-ref/cli-guideline.md | 571 ++++++++++++++++++++
 2 files changed, 572 insertions(+)
 create mode 100644 doc/manual/src/command-ref/cli-guideline.md

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 8281f683f..27a08bb43 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -63,6 +63,7 @@
     - [nix-prefetch-url](command-ref/nix-prefetch-url.md)
   - [Experimental Commands](command-ref/experimental-commands.md)
     - [nix](command-ref/nix.md)
+    - [CLI guideline](command-ref/cli-guideline.md)
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
 - [Glossary](glossary.md)
diff --git a/doc/manual/src/command-ref/cli-guideline.md b/doc/manual/src/command-ref/cli-guideline.md
new file mode 100644
index 000000000..01df07136
--- /dev/null
+++ b/doc/manual/src/command-ref/cli-guideline.md
@@ -0,0 +1,571 @@
+# CLI guideline
+
+## Goals
+
+Purpose of this document is to provide a clear direction to **help design
+delightful command line** experience. This document contain guidelines to
+follow to ensure a consistent and approachable user experience.
+
+## Overview
+
+`nix` command provides a single entry to a number of sub-commands that help
+**developers and system administrators** in the life-cycle of a software
+project. We particularly need to pay special attention to help and assist new
+users of Nix.
+
+# Naming the `COMMANDS`
+
+Words matter. Naming is an important part of the usability. Users will be
+interacting with Nix on a regular basis so we should **name things for ease of
+understanding**.
+
+We recommend following the [Principle of Least
+Astonishment](https://en.wikipedia.org/wiki/Principle_of_least_astonishment).
+This means that you should **never use acronyms or abbreviations** unless they
+are commonly used in other tools (e.g. `nix init`). And if the command name is
+too long (> 10-12 characters) then shortening it makes sense (e.g.
+“prioritization” → “priority”).
+
+Commands should **follow a noun-verb dialogue**. Although noun-verb formatting
+seems backwards from a speaking perspective (i.e. `nix store copy` vs. `nix
+copy store`) it allows us to organize commands the same way users think about
+completing an action (the group first, then the command).
+
+## Naming rules
+
+Rules are there to guide you by limiting your options. But not everything can
+fit the rules all the time. In those cases document the exceptions in [Appendix
+1: Commands naming exceptions](#appendix-1-commands-naming-exceptions) and
+provide reason. The rules want to force a Nix developer to look, not just at
+the command at hand, but also the command in a full context alongside other
+`nix` commands.
+
+```shell
+$ nix [<GROUP>] <COMMAND> [<ARGUMENTS>] [<OPTIONS>]
+```
+
+- `GROUP`, `COMMAND`, `ARGUMENTS` and `OPTIONS` should be lowercase and in a
+  singular form.
+- `GROUP` should be a  **NOUN**.
+- `COMMAND` should be a **VERB**.
+- `ARGUMENTS` and `OPTIONS` are discussed in [*Input* section](#input).
+
+## Classification
+
+Some commands are more important, some less. While we want all of our commands
+to be perfect we can only spend limited amount of time testing and improving
+them.
+
+This classification tries to separate commands in 3 categories in terms of
+their importance in regards to the new users. Users who are likely to be
+impacted the most by bad user experience. This does not mean that we will not
+take care of Tier3 commands, it means we will only put more helpful details for
+beginners into Tier1 commands.
+
+- **Tier1**: Commands used for our main use cases and most likely used by new
+  users. From Tier1 commands we expect attention to details, such as:
+    - Proper use of [colors](#colors), [emojis](#special-unicode-characters)
+      and [aligning of text](#text-alignment).
+    - [Autocomplete](#shell-completion) of options.
+    - Show [next possible steps](#next-steps).
+    - Showing some [“tips”](#educate-the-user) when running logs running tasks
+      (eg. building / downloading) in order to teach users interesting bits of
+      Nix ecosystem.
+    - [Help pages](#help-is-essential) to be as good as we can write them
+      pointing to external documentation and tutorials for more.
+- **Tier2**: Commands that are somewhere between Tier1 and Tier2, not really
+  exposing some implementation detail, but not something that we expect a user.
+  From Tier2 command we expect less attention to details, but still some:
+    - Proper use of [colors](#colors), [emojis](#special-unicode-characters)
+      and [aligning of text](#text-alignment).
+    - [Autocomplete](#shell-completion) of options.
+- **Tier3**: Commands that expose certain internal functionality of `nix`,
+  mostly used by other scripts.
+    - [Autocomplete](#shell-completion) of options.
+
+# Help is essential
+
+Help should be built into your command line so that new users can gradually
+discover new features when they need them. 
+
+## Looking for help
+
+Since there is no standard way how user will look for help we rely on ways help
+is provided by commonly used tools. As a guide for this we took `git` and
+whenever in doubt look at it as a preferred direction.
+
+The rules are:
+
+- Help is shown by using `--help` or `help` command (eg `nix` `--``help` or
+  `nix help`).
+- For non-COMMANDs (eg. `nix` `--``help`  and `nix store` `--``help`) we **show
+  a summary** of most common use cases. Summary is presented on the STDOUT
+  without any use of PAGER.
+- For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the
+  man page of that command. By default the PAGER is used (as in `git`).
+- At the end of either summary or man page there should be an URL pointing to
+  an online version of more detailed documentation.
+- The structure of summaries and man pages should be the same as in `git`.
+
+## Anticipate where help is needed
+
+Even better then requiring the user to search for help is to anticipate and
+predict when user might need it. Either because the lack of discoverability,
+typo in the input or simply taking the opportunity to teach the user of
+interesting - but less visible - details.
+
+### Shell completion
+
+This type of help is most common and almost expected by users. We need to
+**provide the best shell completion** for `bash`, `zsh` and `fish`.
+
+Completion needs to be **context aware**, this mean when a user types:
+
+```shell
+$ nix build n<TAB>
+```
+
+we need to display a list of flakes starting with `n`.
+
+### Wrong input
+
+As we all know we humans make mistakes, all the time. When a typo - intentional
+or unintentional - is made, we should prompt for closest possible options or
+point to the documentation which would educate user to not make the same
+errors. Here are few examples:
+
+In first example we prompt the user for typing wrong command name:
+
+
+```shell
+$ nix int
+------------------------------------------------------------------------
+  Error! Command `int` not found.
+------------------------------------------------------------------------
+  Did you mean:
+    |> nix init
+    |> nix input
+```
+
+Sometimes users will make mistake either because of a typo or simply because of
+lack of discoverability. Our handling of this cases needs to be context
+sensitive.
+
+
+```shell
+$ nix init --template=template#pyton
+------------------------------------------------------------------------
+  Error! Template `template#pyton` not found.
+------------------------------------------------------------------------
+Initializing Nix project at `/path/to/here`.
+      Select a template for you new project:
+          |> template#pyton
+             template#python-pip
+             template#python-poetry
+```
+
+### Next steps
+
+It can be invaluable to newcomers to show what a possible next steps and what
+is the usual development workflow with Nix. For example:
+
+
+```shell
+$ nix init --template=template#python
+Initializing project `template#python`
+          in `/home/USER/dev/new-project`
+
+  Next steps
+    |> nix develop   -- to enter development environment
+    |> nix build     -- to build your project
+```
+
+### Educate the user
+
+We should take any opportunity to **educate users**, but at the same time we
+must **be very very careful to not annoy users**. There is a thin line between
+being helpful and being annoying.
+
+An example of educating users might be to provide *Tips* in places where they
+are waiting.
+
+```shell
+$ nix build
+    Started building my-project 1.2.3
+ Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
+ Downloaded python3.8-requests 1.2.3 in 5.3 seconds
+------------------------------------------------------------------------
+      Press `v` to increase logs verbosity
+         |> `?` to see other options
+------------------------------------------------------------------------
+      Learn something new with every build...
+         |> See last logs of a build with `nix log --last` command.
+------------------------------------------------------------------------
+  Evaluated my-project 1.2.3 in 14.43 seconds
+Downloading [12 / 200]
+         |> firefox 1.2.3 [#########>       ] 10Mb/s | 2min left
+   Building [2 / 20]
+         |> glibc 1.2.3 -> buildPhase: <last log line>
+------------------------------------------------------------------------
+```
+
+Now **Learn** part of the output is where you educate users. You should only
+show it when you know that a build will take some time and not annoy users of
+the builds that take only few seconds.
+
+Every feature like this should go though a intensive review and testing to
+collect as much a feedback as possible and to fine tune every little detail. If
+done right this can be an awesome features beginners and advance users will
+love, but if not done perfectly it will annoy users and leave bad impression.
+
+# Input
+
+Input to a command is provided via `ARGUMENTS` and `OPTIONS`. 
+
+`ARGUMENTS` represent a required input for a function. When choosing to use
+`ARGUMENT` over function please be aware of the downsides that come with it:
+
+- User will need to remember the order of `ARGUMENTS`. This is not a problem if
+  there is only one `ARGUMENT`.
+- With `OPTIONS` it is possible to provide much better auto completion.
+- With `OPTIONS` it is possible to provide much better error message.
+- Using `OPTIONS` it will mean there is a little bit more typing.
+
+We don’t discourage the use of `ARGUMENTS`, but simply want to make every
+developer consider the downsides and choose wisely.
+
+## Naming the `OPTIONS`
+
+Then only naming convention - apart from the ones mentioned in Naming the
+`COMMANDS` section is how flags are named.
+
+Flags are a type of `OPTION` that represent an option that can be turned ON of
+OFF. We can say **flags are boolean type of** `**OPTION**`.
+
+Here are few examples of flag `OPTIONS`:
+
+- `--colors` vs. `--no-colors` (showing colors in the output)
+- `--emojis` vs. `--no-emojis` (showing emojis in the output)
+
+## Prompt when input not provided
+
+For **Tier1** commands we want command to improve the discoverability of
+possible input. A new user will most likely not know which `ARGUMENTS` and
+`OPTIONS` are required or which values are possible for those options.
+
+In cases, the user might not provide the input or they provide wrong input,
+rather then show the error, prompt a user with an option to find and select
+correct input (see examples).
+
+Prompting is of course not required when TTY is not attached to STDIN. This
+would mean that scripts wont need to handle prompt, but rather handle errors.
+
+A place to use prompt and provide user with interactive select
+
+
+```shell
+$ nix init
+Initializing Nix project at `/path/to/here`.
+      Select a template for you new project:
+          |> py
+             template#python-pip
+             template#python-poetry
+             [ Showing 2 templates from 1345 templates ]
+```
+
+Another great place to add prompts are **confirmation dialogues for dangerous
+actions**. For example when adding new substitutor via `OPTIONS` or via
+`flake.nix` we should prompt - for the first time - and let user review what is
+going to happen.
+
+
+```shell
+$ nix build --option substitutors https://cache.example.org
+------------------------------------------------------------------------
+  Warning! A security related question need to be answered.
+------------------------------------------------------------------------
+  The following substitutors will be used to in `my-project`: 
+    - https://cache.example.org
+
+  Do you allow `my-project` to use above mentioned substitutors?
+    [y/N] |> y
+```
+
+# Output
+
+Terminal output can be quite limiting in many ways. Which should forces us to
+think about the experience even more. As with every design the output is a
+compromise between being terse and being verbose, between showing help to
+beginners and annoying advance users. For this it is important that we know
+what are the priorities.
+
+Nix command line should be first and foremost written with beginners in mind.
+But users wont stay beginners for long and what was once useful might quickly
+become annoying. There is no golden rule that we can give in this guideline
+that would make it easier how to draw a line and find best compromise.
+
+What we would encourage is to **build prototypes**, do some **user testing**
+and collect **feedback**. Then repeat the cycle few times.
+
+First design the *happy path* and only after your iron it out, continue to work
+on **edge cases** (handling and displaying errors, changes of the output by
+certain `OPTIONS`, etc…)
+
+## Follow best practices
+
+Needless to say we Nix must be a good citizen and follow best practices in
+command line.
+
+In short: **STDOUT is for output, STDERR is for (human) messaging.**
+
+STDOUT and STDERR provide a way for you to output messages to the user while
+also allowing them to redirect content to a file. For example:
+
+```shell
+$ nix build > build.txt
+------------------------------------------------------------------------
+  Error! Atrribute `bin` missing at (1:94) from string.
+------------------------------------------------------------------------
+
+  1| with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } ""
+```
+
+Because this warning is on STDERR, it doesn’t end up in the file.
+
+But not everything on STDERR is an error though. For example, you can run `nix
+build` and collect logs in a file while still seeing the progress.
+
+```
+$ nix build > build.txt
+  Evaluated 1234 files in 1.2 seconds
+ Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
+ Downloaded python3.8-requests 1.2.3 in 5.3 seconds
+------------------------------------------------------------------------
+      Press `v` to increase logs verbosity
+         |> `?` to see other options
+------------------------------------------------------------------------
+      Learn something new with every build...
+         |> See last logs of a build with `nix log --last` command.
+------------------------------------------------------------------------
+  Evaluated my-project 1.2.3 in 14.43 seconds
+Downloading [12 / 200]
+         |> firefox 1.2.3 [#########>       ] 10Mb/s | 2min left
+   Building [2 / 20]
+         |> glibc 1.2.3 -> buildPhase: <last log line>
+------------------------------------------------------------------------
+```
+
+## Errors (WIP)
+
+**TODO**: Once we have implementation for the *happy path* then we will think
+how to present errors.
+
+## Not only for humans
+
+Terse, machine-readable output formats can also be useful but shouldn’t get in
+the way of making beautiful CLI output. When needed, commands should offer a
+`--json` flag to allow users to easily parse and script the CLI.
+
+When TTY is not detected on STDOUT we should remove all design elements (no
+colors, no emojis and using ASCII instead of Unicode symbols). The same should
+happen when TTY is not detected on STDERR. We should not display progress /
+status section, but only print warnings and errors.
+
+## Dialog with the user
+
+CLIs don't always make it clear when an action has taken place. For every
+action a user performs, your CLI should provide an equal and appropriate
+reaction, clearly highlighting the what just happened. For example:
+
+```shell
+$ nix build
+ Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
+ Downloaded python3.8-requests 1.2.3 in 5.3 seconds
+...
+   Success! You have successfully built my-project.
+$
+```
+
+Above command clearly states that command successfully completed. And in case
+of `nix build`, which is a command that might take some time to complete, it is
+equally important to also show that a command started.
+
+## Text alignment 
+
+Text alignment is the number one design element that will present all of the
+Nix commands as a family and not as separate tools glued together.
+
+The format we should follow is:
+
+```shell
+$ nix COMMAND
+   VERB_1 NOUN and other words
+  VERB__1 NOUN and other words
+       |> Some details 
+```
+
+Few rules that we can extract from above example:
+
+- Each line should start at least with one space.
+- First word should be a VERB and must be aligned to the right.
+- Second word should be a NOUN and must be aligned to the left.
+- If you can not find a good VERB / NOUN pair, don’t worry make it as
+  understandable to the user as possible.
+- More details of each line can be provided by `|>` character which is serving
+  as the first word when aligning the text
+
+Don’t forget you should also test your terminal output with colors and emojis
+off (`--no-colors --no-emojis`).
+
+## Dim / Bright
+
+After comparing few terminals with different color schemes we would **recommend
+to avoid using dimmed text**. The difference from the rest of the text is very
+little in many terminal and color scheme combinations. Sometimes the difference
+is not even notable, therefore relying on it wouldn’t make much sense.
+
+**The bright text is much better supported** across terminals and color
+schemes. Most of the time the difference is perceived as if the bright text
+would be bold. 
+
+## Colors
+
+Humans are already conditioned by society to attach certain meaning to certain
+colors. While the meaning is not universal, a simple collection of colors is
+used to represent basic emotions. 
+
+Colors that can be used in output
+
+- Red = error, danger, stop
+- Green = success, good
+- Yellow/Orange = proceed with caution, warning, in progress
+- Blue/Magenta = stability, calm
+
+While colors are nice, when command line is used by machines (in automation
+scripts) you want to remove the colors. There should be a global `--no-colors`
+option that would remove the colors.
+
+## Special (Unicode) characters
+
+Most of the terminal have good support for Unicode characters and you should
+use them in your output by default. But always have a backup solution that is
+implemented only with ASCII characters and will be used when `--ascii` option
+is going to be passed in. Please make sure that you test your output also
+without Unicode characters
+
+More they showing all the different Unicode characters it is important to
+**establish common set of characters** that we use for certain situations.
+
+## Emojis
+
+Emojis help channel emotions even better than text, colors and special
+characters.
+
+We recommend **keeping the set of emojis to a minimum**. This will enable each
+emoji to stand out more.
+
+As not everybody is happy about emojis we should provide an `--no-emojis`
+option to disable them. Please make sure that you test your output also without
+emojis.
+
+## Tables
+
+All commands that are listing certain data can be implemented in some sort of a
+table. It’s important that each row of your output is a single ‘entry’ of data.
+Never output table borders. It’s noisy and a huge pain for parsing using other
+tools such as `grep`.
+
+Be mindful of the screen width. Only show a few columns by default with the
+table header, for more the table can be manipulated by the following options:
+
+- `--no-headers`: Show column headers by default but allow to hide them.
+- `--columns`: Comma-separated list of column names to add.
+- `--sort`: Allow sorting by column. Allow inverse and multi-column sort as well.
+
+## Interactive output
+
+Interactive output was selected to be able to strike the balance between
+beginners and advance users. While the default output will target beginners it
+can, with a few key strokes, be changed into and advance introspection tool.
+
+### Progress
+
+For longer running commands we should provide and overview of the progress.
+This is shown best in `nix build` example:
+
+```shell
+$ nix build
+    Started building my-project 1.2.3
+ Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
+ Downloaded python3.8-requests 1.2.3 in 5.3 seconds
+------------------------------------------------------------------------
+      Press `v` to increase logs verbosity
+         |> `?` to see other options
+------------------------------------------------------------------------
+      Learn something new with every build...
+         |> See last logs of a build with `nix log --last` command.
+------------------------------------------------------------------------
+  Evaluated my-project 1.2.3 in 14.43 seconds
+Downloading [12 / 200]
+         |> firefox 1.2.3 [#########>       ] 10Mb/s | 2min left
+   Building [2 / 20]
+         |> glibc 1.2.3 -> buildPhase: <last log line>
+------------------------------------------------------------------------
+```
+
+### Search
+
+Use a `fzf` like fuzzy search when there are multiple options to choose from.
+
+```shell
+$ nix init
+Initializing Nix project at `/path/to/here`.
+      Select a template for you new project:
+          |> py
+             template#python-pip
+             template#python-poetry
+             [ Showing 2 templates from 1345 templates ]
+```
+
+### Prompt
+
+In some situations we need to prompt the user and inform the user about what is
+going to happen.
+
+```shell
+$ nix build --option substitutors https://cache.example.org
+------------------------------------------------------------------------
+  Warning! A security related question need to be answered.
+------------------------------------------------------------------------
+  The following substitutors will be used to in `my-project`: 
+    - https://cache.example.org
+
+  Do you allow `my-project` to use above mentioned substitutors?
+    [y/N] |> y
+```
+
+## Verbosity
+
+There are many ways that you can control verbosity.
+
+Verbosity levels are: 
+
+- `ERROR` (level 0)
+- `WARN` (level 1)
+- `NOTICE` (level 2)
+- `INFO` (level 3)
+- `TALKATIVE` (level 4)
+- `CHATTY` (level 5)
+- `DEBUG` (level 6)
+- `VOMIT` (level 7)
+
+The default level that the command starts is `ERROR`. The simplest way to
+increase the verbosity by stacking `-v` option (eg: `-vvv == level 3 == INFO`).
+There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and
+`--quiet` to run in `ERROR` verbosity level.
+
+----------
+
+# Appendix 1: Commands naming exceptions
+
+`nix init` and `nix repl` are well established 

From 148608ba6ddf93168e86525627bed755a474d21f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 12:26:09 +0100
Subject: [PATCH 518/882] Add 'nix help'

---
 src/nix/main.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 5056ceb78..a75f8ae65 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -149,6 +149,50 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     }
 };
 
+static void showHelp(std::vector<std::string> subcommand)
+{
+    showManPage(subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)));
+}
+
+struct CmdHelp : Command
+{
+    std::vector<std::string> subcommand;
+
+    CmdHelp()
+    {
+        expectArgs({
+            .label = "subcommand",
+            .handler = {&subcommand},
+        });
+    }
+
+    std::string description() override
+    {
+        return "show help about 'nix' or a particular subcommand";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To show help about 'nix' in general:",
+                "nix help"
+            },
+            Example{
+                "To show help about a particular subcommand:",
+                "nix help run"
+            },
+        };
+    }
+
+    void run() override
+    {
+        showHelp(subcommand);
+    }
+};
+
+static auto rCmdHelp = registerCommand<CmdHelp>("help");
+
 void mainWrapped(int argc, char * * argv)
 {
     /* The chroot helper needs to be run before any threads have been

From df552a26452f4cdf734edcac049187b8fd806153 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 21:25:32 +0100
Subject: [PATCH 519/882] nix eval: Add option to write a directory

This is useful for generating the nix manpages, but it may have other
applications (like generating configuration files without a Nix store).
---
 src/nix/eval.cc    | 55 ++++++++++++++++++++++++++++++++++++++++++----
 tests/pure-eval.sh |  8 +++++++
 2 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 8da81d667..0f02919de 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -3,6 +3,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "eval.hh"
+#include "eval-inline.hh"
 #include "json.hh"
 #include "value-to-json.hh"
 #include "progress-bar.hh"
@@ -13,6 +14,7 @@ struct CmdEval : MixJSON, InstallableCommand
 {
     bool raw = false;
     std::optional<std::string> apply;
+    std::optional<Path> writeTo;
 
     CmdEval()
     {
@@ -24,6 +26,13 @@ struct CmdEval : MixJSON, InstallableCommand
             .labels = {"expr"},
             .handler = {&apply},
         });
+
+        addFlag({
+            .longName = "write-to",
+            .description = "write a string or attrset of strings to 'path'",
+            .labels = {"path"},
+            .handler = {&writeTo},
+        });
     }
 
     std::string description() override
@@ -66,7 +75,7 @@ struct CmdEval : MixJSON, InstallableCommand
 
         auto state = getEvalState();
 
-        auto v = installable->toValue(*state).first;
+        auto [v, pos] = installable->toValue(*state);
         PathSet context;
 
         if (apply) {
@@ -77,13 +86,51 @@ struct CmdEval : MixJSON, InstallableCommand
             v = vRes;
         }
 
-        if (raw) {
+        if (writeTo) {
+            stopProgressBar();
+
+            if (pathExists(*writeTo))
+                throw Error("path '%s' already exists", *writeTo);
+
+            std::function<void(Value & v, const Pos & pos, const Path & path)> recurse;
+
+            recurse = [&](Value & v, const Pos & pos, const Path & path)
+            {
+                state->forceValue(v);
+                if (v.type == tString)
+                    // FIXME: disallow strings with contexts?
+                    writeFile(path, v.string.s);
+                else if (v.type == tAttrs) {
+                    if (mkdir(path.c_str(), 0777) == -1)
+                        throw SysError("creating directory '%s'", path);
+                    for (auto & attr : *v.attrs)
+                        try {
+                            if (attr.name == "." || attr.name == "..")
+                                throw Error("invalid file name '%s'", attr.name);
+                            recurse(*attr.value, *attr.pos, path + "/" + std::string(attr.name));
+                        } catch (Error & e) {
+                            e.addTrace(*attr.pos, hintfmt("while evaluating the attribute '%s'", attr.name));
+                            throw;
+                        }
+                }
+                else
+                    throw TypeError("value at '%s' is not a string or an attribute set", pos);
+            };
+
+            recurse(*v, pos, *writeTo);
+        }
+
+        else if (raw) {
             stopProgressBar();
             std::cout << state->coerceToString(noPos, *v, context);
-        } else if (json) {
+        }
+
+        else if (json) {
             JSONPlaceholder jsonOut(std::cout);
             printValueAsJSON(*state, true, *v, jsonOut, context);
-        } else {
+        }
+
+        else {
             state->forceValueDeep(*v);
             logger->cout("%s", *v);
         }
diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh
index 43a765997..561ca53fb 100644
--- a/tests/pure-eval.sh
+++ b/tests/pure-eval.sh
@@ -16,3 +16,11 @@ nix eval --expr 'assert 1 + 2 == 3; true'
 [[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]]
 (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x")
 nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x"
+
+rm -rf $TEST_ROOT/eval-out
+nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }'
+[[ $(cat $TEST_ROOT/eval-out/x) = foobar ]]
+[[ $(cat $TEST_ROOT/eval-out/y/z) = bla ]]
+
+rm -rf $TEST_ROOT/eval-out
+(! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }')

From 72428e38d904a1db746f37c8727ab7b0fbb457bc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 23:05:28 +0100
Subject: [PATCH 520/882] Generate separate manpages for each nix subcommand

---
 .gitignore                                   |  3 +-
 doc/manual/generate-manpage.nix              | 50 +++++++++++++-------
 doc/manual/local.mk                          | 28 ++++++++---
 doc/manual/src/{SUMMARY.md => SUMMARY.md.in} |  2 +-
 4 files changed, 58 insertions(+), 25 deletions(-)
 rename doc/manual/src/{SUMMARY.md => SUMMARY.md.in} (99%)

diff --git a/.gitignore b/.gitignore
index c51582cf0..37aada307 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,7 +18,8 @@ perl/Makefile.config
 /doc/manual/nix.json
 /doc/manual/conf-file.json
 /doc/manual/builtins.json
-/doc/manual/src/command-ref/nix.md
+/doc/manual/src/SUMMARY.md
+/doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/expressions/builtins.md
 
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index db266750a..2801e96fa 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -1,34 +1,40 @@
+command:
+
 with builtins;
 with import ./utils.nix;
 
 let
 
   showCommand =
-    { command, section, def }:
-    "${section} Name\n\n"
+    { command, def, filename }:
+    "# Name\n\n"
     + "`${command}` - ${def.description}\n\n"
-    + "${section} Synopsis\n\n"
+    + "# Synopsis\n\n"
     + showSynopsis { inherit command; args = def.args; }
+    + (if def.commands or {} != {}
+       then
+         "where *subcommand* is one of the following:\n\n"
+         # FIXME: group by category
+         + concatStrings (map (name:
+           "* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n")
+           (attrNames def.commands))
+         + "\n"
+       else "")
     + (if def ? doc
-       then "${section} Description\n\n" + def.doc + "\n\n"
+       then "# Description\n\n" + def.doc + "\n\n"
        else "")
     + (let s = showFlags def.flags; in
        if s != ""
-       then "${section} Flags\n\n${s}"
+       then "# Flags\n\n${s}"
        else "")
     + (if def.examples or [] != []
        then
-         "${section} Examples\n\n"
+         "# Examples\n\n"
          + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
-       else "")
-    + (if def.commands or [] != []
-       then concatStrings (
-         map (name:
-           "# Subcommand `${command} ${name}`\n\n"
-           + showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; })
-           (attrNames def.commands))
        else "");
 
+  appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
+
   showFlags = flags:
     concatStrings
       (map (longName:
@@ -48,8 +54,20 @@ let
     "`${command}` [*flags*...] ${concatStringsSep " "
       (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
 
+  processCommand = { command, def, filename }:
+    [ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ]
+    ++ concatMap
+      (name: processCommand {
+        filename = appendName filename name;
+        command = command + " " + name;
+        def = def.commands.${name};
+      })
+      (attrNames def.commands or {});
+
 in
 
-command:
-
-showCommand { command = "nix"; section = "#"; def = command; }
+let
+  manpages = processCommand { filename = "nix"; command = "nix"; def = command; };
+  summary = concatStrings (map (manpage: "    - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
+in
+(listToAttrs manpages) // { "SUMMARY.md" = summary; }
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index bb8b3b60a..b40fa4ed2 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -4,7 +4,7 @@ MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
 
 # Generate man pages.
 man-pages := $(foreach n, \
-  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 nix.1 \
+  nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
   nix-collect-garbage.1 \
   nix-prefetch-url.1 nix-channel.1 \
   nix-hash.1 nix-copy-closure.1 \
@@ -22,7 +22,7 @@ dummy-env = env -i \
 	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
 	NIX_STATE_DIR=/dummy
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
+nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 
 $(d)/%.1: $(d)/src/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
@@ -42,13 +42,17 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	$(trace-gen) lowdown -sT man $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
-	$(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
+	$(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp
 	@mv $@.tmp $@
 
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
+	@rm -rf $@
+	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))'
+
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
@@ -61,7 +65,7 @@ $(d)/conf-file.json: $(bindir)/nix
 
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/builtins.json: $(bindir)/nix
@@ -71,7 +75,17 @@ $(d)/builtins.json: $(bindir)/nix
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/command-ref/nix.md $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
+# Generate 'nix' manpages.
+install: $(d)/src/command-ref/new-cli
+	for i in doc/manual/src/command-ref/new-cli/*.md; do \
+	  name=$$(basename $$i .md); \
+	  if [[ $$name = SUMMARY ]]; then continue; fi; \
+	  printf "Title: %s\n\n" "$$name" > $$i.tmp; \
+	  cat $$i >> $$i.tmp; \
+	  lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \
+	done
+
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
 	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
 	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
 
diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md.in
similarity index 99%
rename from doc/manual/src/SUMMARY.md
rename to doc/manual/src/SUMMARY.md.in
index 8281f683f..b5ae34cfa 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md.in
@@ -62,7 +62,7 @@
     - [nix-instantiate](command-ref/nix-instantiate.md)
     - [nix-prefetch-url](command-ref/nix-prefetch-url.md)
   - [Experimental Commands](command-ref/experimental-commands.md)
-    - [nix](command-ref/nix.md)
+@manpages@
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
 - [Glossary](glossary.md)

From e2efc63979a5485a96accaa1e49ffec65c353078 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Dec 2020 23:19:16 +0100
Subject: [PATCH 521/882] Put examples first in the manpages

---
 doc/manual/generate-manpage.nix | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 2801e96fa..4a0b0290b 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -20,6 +20,11 @@ let
            (attrNames def.commands))
          + "\n"
        else "")
+    + (if def.examples or [] != []
+       then
+         "# Examples\n\n"
+         + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
+       else "")
     + (if def ? doc
        then "# Description\n\n" + def.doc + "\n\n"
        else "")
@@ -27,11 +32,7 @@ let
        if s != ""
        then "# Flags\n\n${s}"
        else "")
-    + (if def.examples or [] != []
-       then
-         "# Examples\n\n"
-         + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
-       else "");
+  ;
 
   appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
 

From b0de7b20160ba1910987c9165bc88689e6587c41 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 2 Dec 2020 17:16:50 -0600
Subject: [PATCH 522/882] Check for rosetta 2 support before installing

---
 scripts/install.in | 5 +++++
 1 file changed, 5 insertions(+)
 mode change 100644 => 100755 scripts/install.in

diff --git a/scripts/install.in b/scripts/install.in
old mode 100644
new mode 100755
index 7c3b795cc..46d3817b9
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -46,6 +46,11 @@ case "$(uname -s).$(uname -m)" in
         system=x86_64-darwin
         ;;
     Darwin.arm64)
+        # check for Rosetta 2 support
+        if ! [ -d /Library/Apple/usr/libexec/oah ]; then
+          oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation"
+        fi
+
         hash=@binaryTarball_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@
         # eventually maybe: arm64-darwin

From addf9f4edeab8e8e2cb115dd98dc764cc1f19fcf Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 2 Dec 2020 19:05:02 -0600
Subject: [PATCH 523/882] Call it aarch64-darwin instead of arm64-darwin
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

gnu-config standardized on aarch64 for machine name so host_cpu part
of $system will always be aarch64. That means system will be
aarch64-darwin too.

uname however could report either “aarch64” (if gnu coreutils) or
“arm64” (if apple’s uname). We should support both for compatiblity
here.
---
 scripts/install.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install.in b/scripts/install.in
index 46d3817b9..9c8831d2b 100755
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -45,7 +45,7 @@ case "$(uname -s).$(uname -m)" in
         path=@tarballPath_x86_64-darwin@
         system=x86_64-darwin
         ;;
-    Darwin.arm64)
+    Darwin.arm64|Darwin.aarch64)
         # check for Rosetta 2 support
         if ! [ -d /Library/Apple/usr/libexec/oah ]; then
           oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation"
@@ -53,7 +53,7 @@ case "$(uname -s).$(uname -m)" in
 
         hash=@binaryTarball_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@
-        # eventually maybe: arm64-darwin
+        # eventually maybe: aarch64-darwin
         system=x86_64-darwin
         ;;
     *) oops "sorry, there is no binary distribution of Nix for your platform";;

From 94f359525ed913635eb21e8599d880db877bcc2a Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 2 Dec 2020 19:14:34 -0600
Subject: [PATCH 524/882] Update config.guess for proper arm64 macOS detection

This fixes results for arm64 macOS so config.guess now reports:

  aarch64-apple-darwin20.1.0

instead of

  arm-apple-darwin20.1.0
---
 config/config.guess |  538 +++++++++----
 config/config.sub   | 1756 ++++++++++++++++++++++---------------------
 2 files changed, 1271 insertions(+), 1023 deletions(-)

diff --git a/config/config.guess b/config/config.guess
index d4fb3213e..699b3a10b 100755
--- a/config/config.guess
+++ b/config/config.guess
@@ -1,8 +1,8 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
-#   Copyright 1992-2018 Free Software Foundation, Inc.
+#   Copyright 1992-2020 Free Software Foundation, Inc.
 
-timestamp='2018-08-02'
+timestamp='2020-11-19'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -27,12 +27,12 @@ timestamp='2018-08-02'
 # Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
 #
 # You can get the latest version of this script from:
-# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
+# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess
 #
 # Please send patches to <config-patches@gnu.org>.
 
 
-me=`echo "$0" | sed -e 's,.*/,,'`
+me=$(echo "$0" | sed -e 's,.*/,,')
 
 usage="\
 Usage: $0 [OPTION]
@@ -50,7 +50,7 @@ version="\
 GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
-Copyright 1992-2018 Free Software Foundation, Inc.
+Copyright 1992-2020 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -96,13 +96,14 @@ fi
 
 tmp=
 # shellcheck disable=SC2172
-trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15
-trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0
+trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15
 
 set_cc_for_build() {
+    # prevent multiple calls if $tmp is already set
+    test "$tmp" && return 0
     : "${TMPDIR=/tmp}"
     # shellcheck disable=SC2039
-    { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
+    { tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } ||
 	{ test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } ||
 	{ tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } ||
 	{ echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; }
@@ -130,16 +131,14 @@ if test -f /.attbin/uname ; then
 	PATH=$PATH:/.attbin ; export PATH
 fi
 
-UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
-UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
-UNAME_SYSTEM=`(uname -s) 2>/dev/null`  || UNAME_SYSTEM=unknown
-UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown
+UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown
+UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown
+UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown
 
 case "$UNAME_SYSTEM" in
 Linux|GNU|GNU/*)
-	# If the system lacks a compiler, then just pick glibc.
-	# We could probably try harder.
-	LIBC=gnu
+	LIBC=unknown
 
 	set_cc_for_build
 	cat <<-EOF > "$dummy.c"
@@ -148,17 +147,29 @@ Linux|GNU|GNU/*)
 	LIBC=uclibc
 	#elif defined(__dietlibc__)
 	LIBC=dietlibc
-	#else
+	#elif defined(__GLIBC__)
 	LIBC=gnu
+	#else
+	#include <stdarg.h>
+	/* First heuristic to detect musl libc.  */
+	#ifdef __DEFINED_va_list
+	LIBC=musl
+	#endif
 	#endif
 	EOF
-	eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`"
+	eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')"
 
-	# If ldd exists, use it to detect musl libc.
-	if command -v ldd >/dev/null && \
-		ldd --version 2>&1 | grep -q ^musl
-	then
-	    LIBC=musl
+	# Second heuristic to detect musl libc.
+	if [ "$LIBC" = unknown ] &&
+	   command -v ldd >/dev/null &&
+	   ldd --version 2>&1 | grep -q ^musl; then
+		LIBC=musl
+	fi
+
+	# If the system lacks a compiler, then just pick glibc.
+	# We could probably try harder.
+	if [ "$LIBC" = unknown ]; then
+		LIBC=gnu
 	fi
 	;;
 esac
@@ -178,19 +189,20 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	# Note: NetBSD doesn't particularly care about the vendor
 	# portion of the name.  We always set it to "unknown".
 	sysctl="sysctl -n hw.machine_arch"
-	UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
+	UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \
 	    "/sbin/$sysctl" 2>/dev/null || \
 	    "/usr/sbin/$sysctl" 2>/dev/null || \
-	    echo unknown)`
+	    echo unknown))
 	case "$UNAME_MACHINE_ARCH" in
+	    aarch64eb) machine=aarch64_be-unknown ;;
 	    armeb) machine=armeb-unknown ;;
 	    arm*) machine=arm-unknown ;;
 	    sh3el) machine=shl-unknown ;;
 	    sh3eb) machine=sh-unknown ;;
 	    sh5el) machine=sh5le-unknown ;;
 	    earmv*)
-		arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
-		endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'`
+		arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,')
+		endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p')
 		machine="${arch}${endian}"-unknown
 		;;
 	    *) machine="$UNAME_MACHINE_ARCH"-unknown ;;
@@ -221,7 +233,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	case "$UNAME_MACHINE_ARCH" in
 	    earm*)
 		expr='s/^earmv[0-9]/-eabi/;s/eb$//'
-		abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"`
+		abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr")
 		;;
 	esac
 	# The OS release
@@ -234,7 +246,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 		release='-gnu'
 		;;
 	    *)
-		release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2`
+		release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2)
 		;;
 	esac
 	# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
@@ -243,15 +255,15 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	echo "$machine-${os}${release}${abi-}"
 	exit ;;
     *:Bitrig:*:*)
-	UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
+	UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//')
 	echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE"
 	exit ;;
     *:OpenBSD:*:*)
-	UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
+	UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//')
 	echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE"
 	exit ;;
     *:LibertyBSD:*:*)
-	UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
+	UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//')
 	echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE"
 	exit ;;
     *:MidnightBSD:*:*)
@@ -263,6 +275,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
     *:SolidBSD:*:*)
 	echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE"
 	exit ;;
+    *:OS108:*:*)
+	echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE"
+	exit ;;
     macppc:MirBSD:*:*)
 	echo powerpc-unknown-mirbsd"$UNAME_RELEASE"
 	exit ;;
@@ -272,26 +287,29 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
     *:Sortix:*:*)
 	echo "$UNAME_MACHINE"-unknown-sortix
 	exit ;;
+    *:Twizzler:*:*)
+	echo "$UNAME_MACHINE"-unknown-twizzler
+	exit ;;
     *:Redox:*:*)
 	echo "$UNAME_MACHINE"-unknown-redox
 	exit ;;
     mips:OSF1:*.*)
-        echo mips-dec-osf1
-        exit ;;
+	echo mips-dec-osf1
+	exit ;;
     alpha:OSF1:*:*)
 	case $UNAME_RELEASE in
 	*4.0)
-		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
+		UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}')
 		;;
 	*5.*)
-		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
+		UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}')
 		;;
 	esac
 	# According to Compaq, /usr/sbin/psrinfo has been available on
 	# OSF/1 and Tru64 systems produced since 1995.  I hope that
 	# covers most systems running today.  This code pipes the CPU
 	# types through head -n 1, so we only detect the type of CPU 0.
-	ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^  The alpha \(.*\) processor.*$/\1/p' | head -n 1`
+	ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^  The alpha \(.*\) processor.*$/\1/p' | head -n 1)
 	case "$ALPHA_CPU_TYPE" in
 	    "EV4 (21064)")
 		UNAME_MACHINE=alpha ;;
@@ -329,7 +347,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	# A Tn.n version is a released field test version.
 	# A Xn.n version is an unreleased experimental baselevel.
 	# 1.2 uses "1.2" for uname -r.
-	echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`"
+	echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)"
 	# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
 	exitcode=$?
 	trap '' 0
@@ -363,7 +381,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	exit ;;
     Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*)
 	# akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE.
-	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+	if test "$( (/bin/universe) 2>/dev/null)" = att ; then
 		echo pyramid-pyramid-sysv3
 	else
 		echo pyramid-pyramid-bsd
@@ -376,54 +394,59 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	echo sparc-icl-nx6
 	exit ;;
     DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*)
-	case `/usr/bin/uname -p` in
+	case $(/usr/bin/uname -p) in
 	    sparc) echo sparc-icl-nx7; exit ;;
 	esac ;;
     s390x:SunOS:*:*)
-	echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
+	echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')"
 	exit ;;
     sun4H:SunOS:5.*:*)
-	echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
+	echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
 	exit ;;
     sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
-	echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
+	echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')"
 	exit ;;
     i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*)
 	echo i386-pc-auroraux"$UNAME_RELEASE"
 	exit ;;
     i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
-	UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
-	case `isainfo -b` in
-	    32)
-		echo i386-pc-solaris2"$UNAME_REL"
-		;;
-	    64)
-		echo x86_64-pc-solaris2"$UNAME_REL"
-		;;
-	esac
+	set_cc_for_build
+	SUN_ARCH=i386
+	# If there is a compiler, see if it is configured for 64-bit objects.
+	# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
+	# This test works for both compilers.
+	if test "$CC_FOR_BUILD" != no_compiler_found; then
+	    if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
+		(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		grep IS_64BIT_ARCH >/dev/null
+	    then
+		SUN_ARCH=x86_64
+	    fi
+	fi
+	echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
 	exit ;;
     sun4*:SunOS:6*:*)
 	# According to config.sub, this is the proper way to canonicalize
 	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
 	# it's likely to be more like Solaris than SunOS4.
-	echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
+	echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
 	exit ;;
     sun4*:SunOS:*:*)
-	case "`/usr/bin/arch -k`" in
+	case "$(/usr/bin/arch -k)" in
 	    Series*|S4*)
-		UNAME_RELEASE=`uname -v`
+		UNAME_RELEASE=$(uname -v)
 		;;
 	esac
 	# Japanese Language versions have a version number like `4.1.3-JL'.
-	echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`"
+	echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')"
 	exit ;;
     sun3*:SunOS:*:*)
 	echo m68k-sun-sunos"$UNAME_RELEASE"
 	exit ;;
     sun*:*:4.2BSD:*)
-	UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
+	UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null)
 	test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3
-	case "`/bin/arch`" in
+	case "$(/bin/arch)" in
 	    sun3)
 		echo m68k-sun-sunos"$UNAME_RELEASE"
 		;;
@@ -503,8 +526,8 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
 	}
 EOF
 	$CC_FOR_BUILD -o "$dummy" "$dummy.c" &&
-	  dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` &&
-	  SYSTEM_NAME=`"$dummy" "$dummyarg"` &&
+	  dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') &&
+	  SYSTEM_NAME=$("$dummy" "$dummyarg") &&
 	    { echo "$SYSTEM_NAME"; exit; }
 	echo mips-mips-riscos"$UNAME_RELEASE"
 	exit ;;
@@ -531,11 +554,11 @@ EOF
 	exit ;;
     AViiON:dgux:*:*)
 	# DG/UX returns AViiON for all architectures
-	UNAME_PROCESSOR=`/usr/bin/uname -p`
-	if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ]
+	UNAME_PROCESSOR=$(/usr/bin/uname -p)
+	if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110
 	then
-	    if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \
-	       [ "$TARGET_BINARY_INTERFACE"x = x ]
+	    if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \
+	       test "$TARGET_BINARY_INTERFACE"x = x
 	    then
 		echo m88k-dg-dgux"$UNAME_RELEASE"
 	    else
@@ -559,17 +582,17 @@ EOF
 	echo m68k-tektronix-bsd
 	exit ;;
     *:IRIX*:*:*)
-	echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`"
+	echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')"
 	exit ;;
     ????????:AIX?:[12].1:2)   # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX.
 	echo romp-ibm-aix     # uname -m gives an 8 hex-code CPU id
-	exit ;;               # Note that: echo "'`uname -s`'" gives 'AIX '
+	exit ;;               # Note that: echo "'$(uname -s)'" gives 'AIX '
     i*86:AIX:*:*)
 	echo i386-ibm-aix
 	exit ;;
     ia64:AIX:*:*)
-	if [ -x /usr/bin/oslevel ] ; then
-		IBM_REV=`/usr/bin/oslevel`
+	if test -x /usr/bin/oslevel ; then
+		IBM_REV=$(/usr/bin/oslevel)
 	else
 		IBM_REV="$UNAME_VERSION.$UNAME_RELEASE"
 	fi
@@ -589,7 +612,7 @@ EOF
 			exit(0);
 			}
 EOF
-		if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"`
+		if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy")
 		then
 			echo "$SYSTEM_NAME"
 		else
@@ -602,15 +625,15 @@ EOF
 	fi
 	exit ;;
     *:AIX:*:[4567])
-	IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
+	IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }')
 	if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then
 		IBM_ARCH=rs6000
 	else
 		IBM_ARCH=powerpc
 	fi
-	if [ -x /usr/bin/lslpp ] ; then
-		IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
-			   awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
+	if test -x /usr/bin/lslpp ; then
+		IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc |
+			   awk -F: '{ print $3 }' | sed s/[0-9]*$/0/)
 	else
 		IBM_REV="$UNAME_VERSION.$UNAME_RELEASE"
 	fi
@@ -638,14 +661,14 @@ EOF
 	echo m68k-hp-bsd4.4
 	exit ;;
     9000/[34678]??:HP-UX:*:*)
-	HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'`
+	HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//')
 	case "$UNAME_MACHINE" in
 	    9000/31?)            HP_ARCH=m68000 ;;
 	    9000/[34]??)         HP_ARCH=m68k ;;
 	    9000/[678][0-9][0-9])
-		if [ -x /usr/bin/getconf ]; then
-		    sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
-		    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
+		if test -x /usr/bin/getconf; then
+		    sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null)
+		    sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null)
 		    case "$sc_cpu_version" in
 		      523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
 		      528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
@@ -657,7 +680,7 @@ EOF
 			esac ;;
 		    esac
 		fi
-		if [ "$HP_ARCH" = "" ]; then
+		if test "$HP_ARCH" = ""; then
 		    set_cc_for_build
 		    sed 's/^		//' << EOF > "$dummy.c"
 
@@ -692,11 +715,11 @@ EOF
 		    exit (0);
 		}
 EOF
-		    (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"`
+		    (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy")
 		    test -z "$HP_ARCH" && HP_ARCH=hppa
 		fi ;;
 	esac
-	if [ "$HP_ARCH" = hppa2.0w ]
+	if test "$HP_ARCH" = hppa2.0w
 	then
 	    set_cc_for_build
 
@@ -720,7 +743,7 @@ EOF
 	echo "$HP_ARCH"-hp-hpux"$HPUX_REV"
 	exit ;;
     ia64:HP-UX:*:*)
-	HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'`
+	HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//')
 	echo ia64-hp-hpux"$HPUX_REV"
 	exit ;;
     3050*:HI-UX:*:*)
@@ -750,7 +773,7 @@ EOF
 	  exit (0);
 	}
 EOF
-	$CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` &&
+	$CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") &&
 		{ echo "$SYSTEM_NAME"; exit; }
 	echo unknown-hitachi-hiuxwe2
 	exit ;;
@@ -770,7 +793,7 @@ EOF
 	echo hppa1.0-hp-osf
 	exit ;;
     i*86:OSF1:*:*)
-	if [ -x /usr/sbin/sysversion ] ; then
+	if test -x /usr/sbin/sysversion ; then
 	    echo "$UNAME_MACHINE"-unknown-osf1mk
 	else
 	    echo "$UNAME_MACHINE"-unknown-osf1
@@ -819,14 +842,14 @@ EOF
 	echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
-	FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
-	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
-	FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'`
+	FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)
+	FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///')
+	FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/')
 	echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     5000:UNIX_System_V:4.*:*)
-	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
-	FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
+	FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///')
+	FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/')
 	echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
@@ -838,26 +861,26 @@ EOF
     *:BSD/OS:*:*)
 	echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE"
 	exit ;;
-    arm*:FreeBSD:*:*)
-	UNAME_PROCESSOR=`uname -p`
+    arm:FreeBSD:*:*)
+	UNAME_PROCESSOR=$(uname -p)
 	set_cc_for_build
 	if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
 	    | grep -q __ARM_PCS_VFP
 	then
-	    echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi
+	    echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi
 	else
-	    echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf
+	    echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf
 	fi
 	exit ;;
     *:FreeBSD:*:*)
-	UNAME_PROCESSOR=`/usr/bin/uname -p`
+	UNAME_PROCESSOR=$(/usr/bin/uname -p)
 	case "$UNAME_PROCESSOR" in
 	    amd64)
 		UNAME_PROCESSOR=x86_64 ;;
 	    i386)
 		UNAME_PROCESSOR=i586 ;;
 	esac
-	echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`"
+	echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')"
 	exit ;;
     i*:CYGWIN*:*)
 	echo "$UNAME_MACHINE"-pc-cygwin
@@ -890,18 +913,18 @@ EOF
 	echo "$UNAME_MACHINE"-pc-uwin
 	exit ;;
     amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*)
-	echo x86_64-unknown-cygwin
+	echo x86_64-pc-cygwin
 	exit ;;
     prep*:SunOS:5.*:*)
-	echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
+	echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
 	exit ;;
     *:GNU:*:*)
 	# the GNU system
-	echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`"
+	echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')"
 	exit ;;
     *:GNU/*:*:*)
 	# other systems with GNU libc and userland
-	echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC"
+	echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC"
 	exit ;;
     *:Minix:*:*)
 	echo "$UNAME_MACHINE"-unknown-minix
@@ -914,7 +937,7 @@ EOF
 	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     alpha:Linux:*:*)
-	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
+	case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in
 	  EV5)   UNAME_MACHINE=alphaev5 ;;
 	  EV56)  UNAME_MACHINE=alphaev56 ;;
 	  PCA56) UNAME_MACHINE=alphapca56 ;;
@@ -981,22 +1004,50 @@ EOF
 	exit ;;
     mips:Linux:*:* | mips64:Linux:*:*)
 	set_cc_for_build
+	IS_GLIBC=0
+	test x"${LIBC}" = xgnu && IS_GLIBC=1
 	sed 's/^	//' << EOF > "$dummy.c"
 	#undef CPU
-	#undef ${UNAME_MACHINE}
-	#undef ${UNAME_MACHINE}el
+	#undef mips
+	#undef mipsel
+	#undef mips64
+	#undef mips64el
+	#if ${IS_GLIBC} && defined(_ABI64)
+	LIBCABI=gnuabi64
+	#else
+	#if ${IS_GLIBC} && defined(_ABIN32)
+	LIBCABI=gnuabin32
+	#else
+	LIBCABI=${LIBC}
+	#endif
+	#endif
+
+	#if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6
+	CPU=mipsisa64r6
+	#else
+	#if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6
+	CPU=mipsisa32r6
+	#else
+	#if defined(__mips64)
+	CPU=mips64
+	#else
+	CPU=mips
+	#endif
+	#endif
+	#endif
+
 	#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
-	CPU=${UNAME_MACHINE}el
+	MIPS_ENDIAN=el
 	#else
 	#if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
-	CPU=${UNAME_MACHINE}
+	MIPS_ENDIAN=
 	#else
-	CPU=
+	MIPS_ENDIAN=
 	#endif
 	#endif
 EOF
-	eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`"
-	test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; }
+	eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')"
+	test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; }
 	;;
     mips64el:Linux:*:*)
 	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
@@ -1015,7 +1066,7 @@ EOF
 	exit ;;
     parisc:Linux:*:* | hppa:Linux:*:*)
 	# Look for CPU level
-	case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
+	case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in
 	  PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;;
 	  PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;;
 	  *)    echo hppa-unknown-linux-"$LIBC" ;;
@@ -1055,7 +1106,17 @@ EOF
 	echo "$UNAME_MACHINE"-dec-linux-"$LIBC"
 	exit ;;
     x86_64:Linux:*:*)
-	echo "$UNAME_MACHINE"-pc-linux-"$LIBC"
+	set_cc_for_build
+	LIBCABI=$LIBC
+	if test "$CC_FOR_BUILD" != no_compiler_found; then
+	    if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \
+		(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		grep IS_X32 >/dev/null
+	    then
+		LIBCABI="$LIBC"x32
+	    fi
+	fi
+	echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI"
 	exit ;;
     xtensa*:Linux:*:*)
 	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
@@ -1095,7 +1156,7 @@ EOF
 	echo "$UNAME_MACHINE"-pc-msdosdjgpp
 	exit ;;
     i*86:*:4.*:*)
-	UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'`
+	UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//')
 	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
 		echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL"
 	else
@@ -1104,19 +1165,19 @@ EOF
 	exit ;;
     i*86:*:5:[678]*)
 	# UnixWare 7.x, OpenUNIX and OpenServer 6.
-	case `/bin/uname -X | grep "^Machine"` in
+	case $(/bin/uname -X | grep "^Machine") in
 	    *486*)	     UNAME_MACHINE=i486 ;;
 	    *Pentium)	     UNAME_MACHINE=i586 ;;
 	    *Pent*|*Celeron) UNAME_MACHINE=i686 ;;
 	esac
-	echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}"
+	echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}"
 	exit ;;
     i*86:*:3.2:*)
 	if test -f /usr/options/cb.name; then
-		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		UNAME_REL=$(sed -n 's/.*Version //p' </usr/options/cb.name)
 		echo "$UNAME_MACHINE"-pc-isc"$UNAME_REL"
 	elif /bin/uname -X 2>/dev/null >/dev/null ; then
-		UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')`
+		UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //'))
 		(/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486
 		(/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \
 			&& UNAME_MACHINE=i586
@@ -1166,7 +1227,7 @@ EOF
     3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0)
 	OS_REL=''
 	test -r /etc/.relid \
-	&& OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
+	&& OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid)
 	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
 	  && { echo i486-ncr-sysv4.3"$OS_REL"; exit; }
 	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
@@ -1177,7 +1238,7 @@ EOF
     NCR*:*:4.2:* | MPRAS*:*:4.2:*)
 	OS_REL='.3'
 	test -r /etc/.relid \
-	    && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
+	    && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid)
 	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
 	    && { echo i486-ncr-sysv4.3"$OS_REL"; exit; }
 	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
@@ -1210,7 +1271,7 @@ EOF
 	exit ;;
     *:SINIX-*:*:*)
 	if uname -p 2>/dev/null >/dev/null ; then
-		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		UNAME_MACHINE=$( (uname -p) 2>/dev/null)
 		echo "$UNAME_MACHINE"-sni-sysv4
 	else
 		echo ns32k-sni-sysv
@@ -1244,7 +1305,7 @@ EOF
 	echo mips-sony-newsos6
 	exit ;;
     R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
-	if [ -d /usr/nec ]; then
+	if test -d /usr/nec; then
 		echo mips-nec-sysv"$UNAME_RELEASE"
 	else
 		echo mips-unknown-sysv"$UNAME_RELEASE"
@@ -1292,44 +1353,48 @@ EOF
     *:Rhapsody:*:*)
 	echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE"
 	exit ;;
+    arm64:Darwin:*:*)
+	echo aarch64-apple-darwin"$UNAME_RELEASE"
+	exit ;;
     *:Darwin:*:*)
-	UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
-	set_cc_for_build
-	if test "$UNAME_PROCESSOR" = unknown ; then
-	    UNAME_PROCESSOR=powerpc
+	UNAME_PROCESSOR=$(uname -p)
+	case $UNAME_PROCESSOR in
+	    unknown) UNAME_PROCESSOR=powerpc ;;
+	esac
+	if command -v xcode-select > /dev/null 2> /dev/null && \
+		! xcode-select --print-path > /dev/null 2> /dev/null ; then
+	    # Avoid executing cc if there is no toolchain installed as
+	    # cc will be a stub that puts up a graphical alert
+	    # prompting the user to install developer tools.
+	    CC_FOR_BUILD=no_compiler_found
+	else
+	    set_cc_for_build
 	fi
-	if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then
-	    if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
-		if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
-		       (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
-		       grep IS_64BIT_ARCH >/dev/null
-		then
-		    case $UNAME_PROCESSOR in
-			i386) UNAME_PROCESSOR=x86_64 ;;
-			powerpc) UNAME_PROCESSOR=powerpc64 ;;
-		    esac
-		fi
-		# On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc
-		if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \
-		       (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
-		       grep IS_PPC >/dev/null
-		then
-		    UNAME_PROCESSOR=powerpc
-		fi
+	if test "$CC_FOR_BUILD" != no_compiler_found; then
+	    if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
+		   (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		   grep IS_64BIT_ARCH >/dev/null
+	    then
+		case $UNAME_PROCESSOR in
+		    i386) UNAME_PROCESSOR=x86_64 ;;
+		    powerpc) UNAME_PROCESSOR=powerpc64 ;;
+		esac
+	    fi
+	    # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc
+	    if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \
+		   (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		   grep IS_PPC >/dev/null
+	    then
+		UNAME_PROCESSOR=powerpc
 	    fi
 	elif test "$UNAME_PROCESSOR" = i386 ; then
-	    # Avoid executing cc on OS X 10.9, as it ships with a stub
-	    # that puts up a graphical alert prompting to install
-	    # developer tools.  Any system running Mac OS X 10.7 or
-	    # later (Darwin 11 and later) is required to have a 64-bit
-	    # processor. This is not true of the ARM version of Darwin
-	    # that Apple uses in portable devices.
-	    UNAME_PROCESSOR=x86_64
+	    # uname -m returns i386 or x86_64
+	    UNAME_PROCESSOR=$UNAME_MACHINE
 	fi
 	echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE"
 	exit ;;
     *:procnto*:*:* | *:QNX:[0123456789]*:*)
-	UNAME_PROCESSOR=`uname -p`
+	UNAME_PROCESSOR=$(uname -p)
 	if test "$UNAME_PROCESSOR" = x86; then
 		UNAME_PROCESSOR=i386
 		UNAME_MACHINE=pc
@@ -1397,10 +1462,10 @@ EOF
 	echo mips-sei-seiux"$UNAME_RELEASE"
 	exit ;;
     *:DragonFly:*:*)
-	echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`"
+	echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')"
 	exit ;;
     *:*VMS:*:*)
-	UNAME_MACHINE=`(uname -p) 2>/dev/null`
+	UNAME_MACHINE=$( (uname -p) 2>/dev/null)
 	case "$UNAME_MACHINE" in
 	    A*) echo alpha-dec-vms ; exit ;;
 	    I*) echo ia64-dec-vms ; exit ;;
@@ -1410,7 +1475,7 @@ EOF
 	echo i386-pc-xenix
 	exit ;;
     i*86:skyos:*:*)
-	echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`"
+	echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')"
 	exit ;;
     i*86:rdos:*:*)
 	echo "$UNAME_MACHINE"-pc-rdos
@@ -1424,8 +1489,148 @@ EOF
     amd64:Isilon\ OneFS:*:*)
 	echo x86_64-unknown-onefs
 	exit ;;
+    *:Unleashed:*:*)
+	echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE"
+	exit ;;
 esac
 
+# No uname command or uname output not recognized.
+set_cc_for_build
+cat > "$dummy.c" <<EOF
+#ifdef _SEQUENT_
+#include <sys/types.h>
+#include <sys/utsname.h>
+#endif
+#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__)
+#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__)
+#include <signal.h>
+#if defined(_SIZE_T_) || defined(SIGLOST)
+#include <sys/utsname.h>
+#endif
+#endif
+#endif
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+#include <sys/param.h>
+  printf ("m68k-sony-newsos%s\n",
+#ifdef NEWSOS4
+  "4"
+#else
+  ""
+#endif
+  ); exit (0);
+#endif
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null);
+  if (version < 4)
+    printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
+  else
+    printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-pc-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  struct utsname un;
+
+  uname(&un);
+  if (strncmp(un.version, "V2", 2) == 0) {
+    printf ("i386-sequent-ptx2\n"); exit (0);
+  }
+  if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
+    printf ("i386-sequent-ptx1\n"); exit (0);
+  }
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+#include <sys/param.h>
+#if defined (BSD)
+#if BSD == 43
+  printf ("vax-dec-bsd4.3\n"); exit (0);
+#else
+#if BSD == 199006
+  printf ("vax-dec-bsd4.3reno\n"); exit (0);
+#else
+  printf ("vax-dec-bsd\n"); exit (0);
+#endif
+#endif
+#else
+  printf ("vax-dec-bsd\n"); exit (0);
+#endif
+#else
+#if defined(_SIZE_T_) || defined(SIGLOST)
+  struct utsname un;
+  uname (&un);
+  printf ("vax-dec-ultrix%s\n", un.release); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+#endif
+#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__)
+#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__)
+#if defined(_SIZE_T_) || defined(SIGLOST)
+  struct utsname *un;
+  uname (&un);
+  printf ("mips-dec-ultrix%s\n", un.release); exit (0);
+#else
+  printf ("mips-dec-ultrix\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) &&
+	{ echo "$SYSTEM_NAME"; exit; }
+
+# Apollos put the system type in the environment.
+test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; }
+
 echo "$0: unable to guess system type" >&2
 
 case "$UNAME_MACHINE:$UNAME_SYSTEM" in
@@ -1445,9 +1650,15 @@ This script (version $timestamp), has failed to recognize the
 operating system you are using. If your script is old, overwrite *all*
 copies of config.guess and config.sub with the latest versions from:
 
-  https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
+  https://git.savannah.gnu.org/cgit/config.git/plain/config.guess
 and
-  https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
+  https://git.savannah.gnu.org/cgit/config.git/plain/config.sub
+EOF
+
+year=$(echo $timestamp | sed 's,-.*,,')
+# shellcheck disable=SC2003
+if test "$(expr "$(date +%Y)" - "$year")" -lt 3 ; then
+   cat >&2 <<EOF
 
 If $0 has already been updated, send the following data and any
 information you think might be pertinent to config-patches@gnu.org to
@@ -1455,26 +1666,27 @@ provide the necessary information to handle your system.
 
 config.guess timestamp = $timestamp
 
-uname -m = `(uname -m) 2>/dev/null || echo unknown`
-uname -r = `(uname -r) 2>/dev/null || echo unknown`
-uname -s = `(uname -s) 2>/dev/null || echo unknown`
-uname -v = `(uname -v) 2>/dev/null || echo unknown`
+uname -m = $( (uname -m) 2>/dev/null || echo unknown)
+uname -r = $( (uname -r) 2>/dev/null || echo unknown)
+uname -s = $( (uname -s) 2>/dev/null || echo unknown)
+uname -v = $( (uname -v) 2>/dev/null || echo unknown)
 
-/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null`
-/bin/uname -X     = `(/bin/uname -X) 2>/dev/null`
+/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null)
+/bin/uname -X     = $( (/bin/uname -X) 2>/dev/null)
 
-hostinfo               = `(hostinfo) 2>/dev/null`
-/bin/universe          = `(/bin/universe) 2>/dev/null`
-/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null`
-/bin/arch              = `(/bin/arch) 2>/dev/null`
-/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null`
-/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null`
+hostinfo               = $( (hostinfo) 2>/dev/null)
+/bin/universe          = $( (/bin/universe) 2>/dev/null)
+/usr/bin/arch -k       = $( (/usr/bin/arch -k) 2>/dev/null)
+/bin/arch              = $( (/bin/arch) 2>/dev/null)
+/usr/bin/oslevel       = $( (/usr/bin/oslevel) 2>/dev/null)
+/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null)
 
 UNAME_MACHINE = "$UNAME_MACHINE"
 UNAME_RELEASE = "$UNAME_RELEASE"
 UNAME_SYSTEM  = "$UNAME_SYSTEM"
 UNAME_VERSION = "$UNAME_VERSION"
 EOF
+fi
 
 exit 1
 
diff --git a/config/config.sub b/config/config.sub
index c19e67180..19c9553b1 100755
--- a/config/config.sub
+++ b/config/config.sub
@@ -1,8 +1,8 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright 1992-2018 Free Software Foundation, Inc.
+#   Copyright 1992-2020 Free Software Foundation, Inc.
 
-timestamp='2018-08-13'
+timestamp='2020-12-02'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ timestamp='2018-08-13'
 # Otherwise, we print the canonical config type on stdout and succeed.
 
 # You can get the latest version of this script from:
-# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
+# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub
 
 # This file is supposed to be the same for all GNU packages
 # and recognize all the CPU types, system types and aliases
@@ -50,7 +50,7 @@ timestamp='2018-08-13'
 #	CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM
 # It is wrong to echo any other type of specification.
 
-me=`echo "$0" | sed -e 's,.*/,,'`
+me=$(echo "$0" | sed -e 's,.*/,,')
 
 usage="\
 Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS
@@ -67,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright 1992-2018 Free Software Foundation, Inc.
+Copyright 1992-2020 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -89,7 +89,7 @@ while test $# -gt 0 ; do
     - )	# Use stdin as input.
        break ;;
     -* )
-       echo "$me: invalid option $1$help"
+       echo "$me: invalid option $1$help" >&2
        exit 1 ;;
 
     *local*)
@@ -111,7 +111,8 @@ case $# in
 esac
 
 # Split fields of configuration type
-IFS="-" read -r field1 field2 field3 field4 <<EOF
+# shellcheck disable=SC2162
+IFS="-" read field1 field2 field3 field4 <<EOF
 $1
 EOF
 
@@ -123,55 +124,64 @@ case $1 in
 		;;
 	*-*-*-*)
 		basic_machine=$field1-$field2
-		os=$field3-$field4
+		basic_os=$field3-$field4
 		;;
 	*-*-*)
 		# Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two
 		# parts
 		maybe_os=$field2-$field3
 		case $maybe_os in
-			nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \
-			| linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \
+			nto-qnx* | linux-* | uclinux-uclibc* \
 			| uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \
 			| netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \
 			| storm-chaos* | os2-emx* | rtmk-nova*)
 				basic_machine=$field1
-				os=$maybe_os
+				basic_os=$maybe_os
 				;;
 			android-linux)
 				basic_machine=$field1-unknown
-				os=linux-android
+				basic_os=linux-android
 				;;
 			*)
 				basic_machine=$field1-$field2
-				os=$field3
+				basic_os=$field3
 				;;
 		esac
 		;;
 	*-*)
-		# Second component is usually, but not always the OS
-		case $field2 in
-			# Prevent following clause from handling this valid os
-			sun*os*)
-				basic_machine=$field1
-				os=$field2
-				;;
-			# Manufacturers
-			dec* | mips* | sequent* | encore* | pc532* | sgi* | sony* \
-			| att* | 7300* | 3300* | delta* | motorola* | sun[234]* \
-			| unicom* | ibm* | next | hp | isi* | apollo | altos* \
-			| convergent* | ncr* | news | 32* | 3600* | 3100* | hitachi* \
-			| c[123]* | convex* | sun | crds | omron* | dg | ultra | tti* \
-			| harris | dolphin | highlevel | gould | cbm | ns | masscomp \
-			| apple | axis | knuth | cray | microblaze* \
-			| sim | cisco | oki | wec | wrs | winbond)
-				basic_machine=$field1-$field2
-				os=
-				;;
-			*)
-				basic_machine=$field1
-				os=$field2
+		# A lone config we happen to match not fitting any pattern
+		case $field1-$field2 in
+			decstation-3100)
+				basic_machine=mips-dec
+				basic_os=
 				;;
+			*-*)
+				# Second component is usually, but not always the OS
+				case $field2 in
+					# Prevent following clause from handling this valid os
+					sun*os*)
+						basic_machine=$field1
+						basic_os=$field2
+						;;
+					# Manufacturers
+					dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \
+					| att* | 7300* | 3300* | delta* | motorola* | sun[234]* \
+					| unicom* | ibm* | next | hp | isi* | apollo | altos* \
+					| convergent* | ncr* | news | 32* | 3600* | 3100* \
+					| hitachi* | c[123]* | convex* | sun | crds | omron* | dg \
+					| ultra | tti* | harris | dolphin | highlevel | gould \
+					| cbm | ns | masscomp | apple | axis | knuth | cray \
+					| microblaze* | sim | cisco \
+					| oki | wec | wrs | winbond)
+						basic_machine=$field1-$field2
+						basic_os=
+						;;
+					*)
+						basic_machine=$field1
+						basic_os=$field2
+						;;
+				esac
+			;;
 		esac
 		;;
 	*)
@@ -180,1115 +190,1088 @@ case $1 in
 		case $field1 in
 			386bsd)
 				basic_machine=i386-pc
-				os=bsd
+				basic_os=bsd
 				;;
 			a29khif)
 				basic_machine=a29k-amd
-				os=udi
+				basic_os=udi
 				;;
 			adobe68k)
 				basic_machine=m68010-adobe
-				os=scout
+				basic_os=scout
 				;;
 			alliant)
 				basic_machine=fx80-alliant
-				os=
+				basic_os=
 				;;
 			altos | altos3068)
 				basic_machine=m68k-altos
-				os=
+				basic_os=
 				;;
 			am29k)
 				basic_machine=a29k-none
-				os=bsd
+				basic_os=bsd
 				;;
 			amdahl)
 				basic_machine=580-amdahl
-				os=sysv
+				basic_os=sysv
+				;;
+			amiga)
+				basic_machine=m68k-unknown
+				basic_os=
 				;;
 			amigaos | amigados)
 				basic_machine=m68k-unknown
-				os=amigaos
+				basic_os=amigaos
 				;;
 			amigaunix | amix)
 				basic_machine=m68k-unknown
-				os=sysv4
+				basic_os=sysv4
 				;;
 			apollo68)
 				basic_machine=m68k-apollo
-				os=sysv
+				basic_os=sysv
 				;;
 			apollo68bsd)
 				basic_machine=m68k-apollo
-				os=bsd
+				basic_os=bsd
 				;;
 			aros)
 				basic_machine=i386-pc
-				os=aros
+				basic_os=aros
 				;;
 			aux)
 				basic_machine=m68k-apple
-				os=aux
+				basic_os=aux
 				;;
 			balance)
 				basic_machine=ns32k-sequent
-				os=dynix
+				basic_os=dynix
 				;;
 			blackfin)
 				basic_machine=bfin-unknown
-				os=linux
+				basic_os=linux
 				;;
 			cegcc)
 				basic_machine=arm-unknown
-				os=cegcc
+				basic_os=cegcc
 				;;
 			convex-c1)
 				basic_machine=c1-convex
-				os=bsd
+				basic_os=bsd
 				;;
 			convex-c2)
 				basic_machine=c2-convex
-				os=bsd
+				basic_os=bsd
 				;;
 			convex-c32)
 				basic_machine=c32-convex
-				os=bsd
+				basic_os=bsd
 				;;
 			convex-c34)
 				basic_machine=c34-convex
-				os=bsd
+				basic_os=bsd
 				;;
 			convex-c38)
 				basic_machine=c38-convex
-				os=bsd
+				basic_os=bsd
 				;;
 			cray)
 				basic_machine=j90-cray
-				os=unicos
+				basic_os=unicos
 				;;
 			crds | unos)
 				basic_machine=m68k-crds
-				os=
+				basic_os=
+				;;
+			da30)
+				basic_machine=m68k-da30
+				basic_os=
+				;;
+			decstation | pmax | pmin | dec3100 | decstatn)
+				basic_machine=mips-dec
+				basic_os=
 				;;
 			delta88)
 				basic_machine=m88k-motorola
-				os=sysv3
+				basic_os=sysv3
 				;;
 			dicos)
 				basic_machine=i686-pc
-				os=dicos
+				basic_os=dicos
 				;;
 			djgpp)
 				basic_machine=i586-pc
-				os=msdosdjgpp
+				basic_os=msdosdjgpp
 				;;
 			ebmon29k)
 				basic_machine=a29k-amd
-				os=ebmon
+				basic_os=ebmon
 				;;
 			es1800 | OSE68k | ose68k | ose | OSE)
 				basic_machine=m68k-ericsson
-				os=ose
+				basic_os=ose
 				;;
 			gmicro)
 				basic_machine=tron-gmicro
-				os=sysv
+				basic_os=sysv
 				;;
 			go32)
 				basic_machine=i386-pc
-				os=go32
+				basic_os=go32
 				;;
 			h8300hms)
 				basic_machine=h8300-hitachi
-				os=hms
+				basic_os=hms
 				;;
 			h8300xray)
 				basic_machine=h8300-hitachi
-				os=xray
+				basic_os=xray
 				;;
 			h8500hms)
 				basic_machine=h8500-hitachi
-				os=hms
+				basic_os=hms
 				;;
 			harris)
 				basic_machine=m88k-harris
-				os=sysv3
+				basic_os=sysv3
+				;;
+			hp300 | hp300hpux)
+				basic_machine=m68k-hp
+				basic_os=hpux
 				;;
 			hp300bsd)
 				basic_machine=m68k-hp
-				os=bsd
-				;;
-			hp300hpux)
-				basic_machine=m68k-hp
-				os=hpux
+				basic_os=bsd
 				;;
 			hppaosf)
 				basic_machine=hppa1.1-hp
-				os=osf
+				basic_os=osf
 				;;
 			hppro)
 				basic_machine=hppa1.1-hp
-				os=proelf
+				basic_os=proelf
 				;;
 			i386mach)
 				basic_machine=i386-mach
-				os=mach
-				;;
-			vsta)
-				basic_machine=i386-pc
-				os=vsta
+				basic_os=mach
 				;;
 			isi68 | isi)
 				basic_machine=m68k-isi
-				os=sysv
+				basic_os=sysv
 				;;
 			m68knommu)
 				basic_machine=m68k-unknown
-				os=linux
+				basic_os=linux
 				;;
 			magnum | m3230)
 				basic_machine=mips-mips
-				os=sysv
+				basic_os=sysv
 				;;
 			merlin)
 				basic_machine=ns32k-utek
-				os=sysv
+				basic_os=sysv
 				;;
 			mingw64)
 				basic_machine=x86_64-pc
-				os=mingw64
+				basic_os=mingw64
 				;;
 			mingw32)
 				basic_machine=i686-pc
-				os=mingw32
+				basic_os=mingw32
 				;;
 			mingw32ce)
 				basic_machine=arm-unknown
-				os=mingw32ce
+				basic_os=mingw32ce
 				;;
 			monitor)
 				basic_machine=m68k-rom68k
-				os=coff
+				basic_os=coff
 				;;
 			morphos)
 				basic_machine=powerpc-unknown
-				os=morphos
+				basic_os=morphos
 				;;
 			moxiebox)
 				basic_machine=moxie-unknown
-				os=moxiebox
+				basic_os=moxiebox
 				;;
 			msdos)
 				basic_machine=i386-pc
-				os=msdos
+				basic_os=msdos
 				;;
 			msys)
 				basic_machine=i686-pc
-				os=msys
+				basic_os=msys
 				;;
 			mvs)
 				basic_machine=i370-ibm
-				os=mvs
+				basic_os=mvs
 				;;
 			nacl)
 				basic_machine=le32-unknown
-				os=nacl
+				basic_os=nacl
 				;;
 			ncr3000)
 				basic_machine=i486-ncr
-				os=sysv4
+				basic_os=sysv4
 				;;
 			netbsd386)
 				basic_machine=i386-pc
-				os=netbsd
+				basic_os=netbsd
 				;;
 			netwinder)
 				basic_machine=armv4l-rebel
-				os=linux
+				basic_os=linux
 				;;
 			news | news700 | news800 | news900)
 				basic_machine=m68k-sony
-				os=newsos
+				basic_os=newsos
 				;;
 			news1000)
 				basic_machine=m68030-sony
-				os=newsos
+				basic_os=newsos
 				;;
 			necv70)
 				basic_machine=v70-nec
-				os=sysv
+				basic_os=sysv
 				;;
 			nh3000)
 				basic_machine=m68k-harris
-				os=cxux
+				basic_os=cxux
 				;;
 			nh[45]000)
 				basic_machine=m88k-harris
-				os=cxux
+				basic_os=cxux
 				;;
 			nindy960)
 				basic_machine=i960-intel
-				os=nindy
+				basic_os=nindy
 				;;
 			mon960)
 				basic_machine=i960-intel
-				os=mon960
+				basic_os=mon960
 				;;
 			nonstopux)
 				basic_machine=mips-compaq
-				os=nonstopux
+				basic_os=nonstopux
 				;;
 			os400)
 				basic_machine=powerpc-ibm
-				os=os400
+				basic_os=os400
 				;;
 			OSE68000 | ose68000)
 				basic_machine=m68000-ericsson
-				os=ose
+				basic_os=ose
 				;;
 			os68k)
 				basic_machine=m68k-none
-				os=os68k
+				basic_os=os68k
 				;;
 			paragon)
 				basic_machine=i860-intel
-				os=osf
+				basic_os=osf
 				;;
 			parisc)
 				basic_machine=hppa-unknown
-				os=linux
+				basic_os=linux
+				;;
+			psp)
+				basic_machine=mipsallegrexel-sony
+				basic_os=psp
 				;;
 			pw32)
 				basic_machine=i586-unknown
-				os=pw32
+				basic_os=pw32
 				;;
 			rdos | rdos64)
 				basic_machine=x86_64-pc
-				os=rdos
+				basic_os=rdos
 				;;
 			rdos32)
 				basic_machine=i386-pc
-				os=rdos
+				basic_os=rdos
 				;;
 			rom68k)
 				basic_machine=m68k-rom68k
-				os=coff
+				basic_os=coff
 				;;
 			sa29200)
 				basic_machine=a29k-amd
-				os=udi
+				basic_os=udi
 				;;
 			sei)
 				basic_machine=mips-sei
-				os=seiux
+				basic_os=seiux
+				;;
+			sequent)
+				basic_machine=i386-sequent
+				basic_os=
 				;;
 			sps7)
 				basic_machine=m68k-bull
-				os=sysv2
+				basic_os=sysv2
 				;;
 			st2000)
 				basic_machine=m68k-tandem
-				os=
+				basic_os=
 				;;
 			stratus)
 				basic_machine=i860-stratus
-				os=sysv4
+				basic_os=sysv4
 				;;
 			sun2)
 				basic_machine=m68000-sun
-				os=
+				basic_os=
 				;;
 			sun2os3)
 				basic_machine=m68000-sun
-				os=sunos3
+				basic_os=sunos3
 				;;
 			sun2os4)
 				basic_machine=m68000-sun
-				os=sunos4
+				basic_os=sunos4
 				;;
 			sun3)
 				basic_machine=m68k-sun
-				os=
+				basic_os=
 				;;
 			sun3os3)
 				basic_machine=m68k-sun
-				os=sunos3
+				basic_os=sunos3
 				;;
 			sun3os4)
 				basic_machine=m68k-sun
-				os=sunos4
+				basic_os=sunos4
 				;;
 			sun4)
 				basic_machine=sparc-sun
-				os=
+				basic_os=
 				;;
 			sun4os3)
 				basic_machine=sparc-sun
-				os=sunos3
+				basic_os=sunos3
 				;;
 			sun4os4)
 				basic_machine=sparc-sun
-				os=sunos4
+				basic_os=sunos4
 				;;
 			sun4sol2)
 				basic_machine=sparc-sun
-				os=solaris2
+				basic_os=solaris2
 				;;
 			sun386 | sun386i | roadrunner)
 				basic_machine=i386-sun
-				os=
+				basic_os=
 				;;
 			sv1)
 				basic_machine=sv1-cray
-				os=unicos
+				basic_os=unicos
 				;;
 			symmetry)
 				basic_machine=i386-sequent
-				os=dynix
+				basic_os=dynix
 				;;
 			t3e)
 				basic_machine=alphaev5-cray
-				os=unicos
+				basic_os=unicos
 				;;
 			t90)
 				basic_machine=t90-cray
-				os=unicos
+				basic_os=unicos
 				;;
 			toad1)
 				basic_machine=pdp10-xkl
-				os=tops20
+				basic_os=tops20
 				;;
 			tpf)
 				basic_machine=s390x-ibm
-				os=tpf
+				basic_os=tpf
 				;;
 			udi29k)
 				basic_machine=a29k-amd
-				os=udi
+				basic_os=udi
 				;;
 			ultra3)
 				basic_machine=a29k-nyu
-				os=sym1
+				basic_os=sym1
 				;;
 			v810 | necv810)
 				basic_machine=v810-nec
-				os=none
+				basic_os=none
 				;;
 			vaxv)
 				basic_machine=vax-dec
-				os=sysv
+				basic_os=sysv
 				;;
 			vms)
 				basic_machine=vax-dec
-				os=vms
+				basic_os=vms
+				;;
+			vsta)
+				basic_machine=i386-pc
+				basic_os=vsta
 				;;
 			vxworks960)
 				basic_machine=i960-wrs
-				os=vxworks
+				basic_os=vxworks
 				;;
 			vxworks68)
 				basic_machine=m68k-wrs
-				os=vxworks
+				basic_os=vxworks
 				;;
 			vxworks29k)
 				basic_machine=a29k-wrs
-				os=vxworks
+				basic_os=vxworks
 				;;
 			xbox)
 				basic_machine=i686-pc
-				os=mingw32
+				basic_os=mingw32
 				;;
 			ymp)
 				basic_machine=ymp-cray
-				os=unicos
+				basic_os=unicos
 				;;
 			*)
 				basic_machine=$1
-				os=
+				basic_os=
 				;;
 		esac
 		;;
 esac
 
-# Decode aliases for certain CPU-COMPANY combinations.
+# Decode 1-component or ad-hoc basic machines
 case $basic_machine in
 	# Here we handle the default manufacturer of certain CPU types.  It is in
 	# some cases the only manufacturer, in others, it is the most popular.
-	craynv)
-		basic_machine=craynv-cray
-		os=${os:-unicosmp}
-		;;
-	fx80)
-		basic_machine=fx80-alliant
-		;;
 	w89k)
-		basic_machine=hppa1.1-winbond
+		cpu=hppa1.1
+		vendor=winbond
 		;;
 	op50n)
-		basic_machine=hppa1.1-oki
+		cpu=hppa1.1
+		vendor=oki
 		;;
 	op60c)
-		basic_machine=hppa1.1-oki
+		cpu=hppa1.1
+		vendor=oki
 		;;
-	romp)
-		basic_machine=romp-ibm
-		;;
-	mmix)
-		basic_machine=mmix-knuth
-		;;
-	rs6000)
-		basic_machine=rs6000-ibm
-		;;
-	vax)
-		basic_machine=vax-dec
-		;;
-	pdp11)
-		basic_machine=pdp11-dec
-		;;
-	we32k)
-		basic_machine=we32k-att
-		;;
-	cydra)
-		basic_machine=cydra-cydrome
-		;;
-	i370-ibm* | ibm*)
-		basic_machine=i370-ibm
-		;;
-	orion)
-		basic_machine=orion-highlevel
+	ibm*)
+		cpu=i370
+		vendor=ibm
 		;;
 	orion105)
-		basic_machine=clipper-highlevel
+		cpu=clipper
+		vendor=highlevel
 		;;
 	mac | mpw | mac-mpw)
-		basic_machine=m68k-apple
+		cpu=m68k
+		vendor=apple
 		;;
 	pmac | pmac-mpw)
-		basic_machine=powerpc-apple
-		;;
-	xps | xps100)
-		basic_machine=xps100-honeywell
+		cpu=powerpc
+		vendor=apple
 		;;
 
-	# Recognize the basic CPU types without company name.
-	# Some are omitted here because they have special meanings below.
-	1750a | 580 \
-	| a29k \
-	| aarch64 | aarch64_be \
-	| abacus \
-	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
-	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
-	| am33_2.0 \
-	| arc | arceb \
-	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \
-	| avr | avr32 \
-	| asmjs \
-	| ba \
-	| be32 | be64 \
-	| bfin \
-	| c4x | c8051 | clipper | csky \
-	| d10v | d30v | dlx | dsp16xx \
-	| e2k | epiphany \
-	| fido | fr30 | frv | ft32 \
-	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
-	| hexagon \
-	| i370 | i860 | i960 | ia16 | ia64 \
-	| ip2k | iq2000 \
-	| k1om \
-	| le32 | le64 \
-	| lm32 \
-	| m32c | m32r | m32rle | m68000 | m68k | m88k \
-	| m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip \
-	| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
-	| mips | mipsbe | mipseb | mipsel | mipsle \
-	| mips16 \
-	| mips64 | mips64el \
-	| mips64octeon | mips64octeonel \
-	| mips64orion | mips64orionel \
-	| mips64r5900 | mips64r5900el \
-	| mips64vr | mips64vrel \
-	| mips64vr4100 | mips64vr4100el \
-	| mips64vr4300 | mips64vr4300el \
-	| mips64vr5000 | mips64vr5000el \
-	| mips64vr5900 | mips64vr5900el \
-	| mipsisa32 | mipsisa32el \
-	| mipsisa32r2 | mipsisa32r2el \
-	| mipsisa32r6 | mipsisa32r6el \
-	| mipsisa64 | mipsisa64el \
-	| mipsisa64r2 | mipsisa64r2el \
-	| mipsisa64r6 | mipsisa64r6el \
-	| mipsisa64sb1 | mipsisa64sb1el \
-	| mipsisa64sr71k | mipsisa64sr71kel \
-	| mipsr5900 | mipsr5900el \
-	| mipstx39 | mipstx39el \
-	| mn10200 | mn10300 \
-	| moxie \
-	| mt \
-	| msp430 \
-	| nds32 | nds32le | nds32be \
-	| nfp \
-	| nios | nios2 | nios2eb | nios2el \
-	| ns16k | ns32k \
-	| open8 | or1k | or1knd | or32 \
-	| pdp10 | pj | pjl \
-	| powerpc | powerpc64 | powerpc64le | powerpcle \
-	| pru \
-	| pyramid \
-	| riscv | riscv32 | riscv64 \
-	| rl78 | rx \
-	| score \
-	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh[23]ele \
-	| sh64 | sh64le \
-	| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
-	| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
-	| spu \
-	| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
-	| ubicom32 \
-	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
-	| visium \
-	| wasm32 \
-	| x86 | xc16x | xstormy16 | xgate | xtensa \
-	| z8k | z80)
-		basic_machine=$basic_machine-unknown
-		;;
-	c54x)
-		basic_machine=tic54x-unknown
-		;;
-	c55x)
-		basic_machine=tic55x-unknown
-		;;
-	c6x)
-		basic_machine=tic6x-unknown
-		;;
-	leon|leon[3-9])
-		basic_machine=sparc-$basic_machine
-		;;
-	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65)
-		;;
-	m9s12z | m68hcs12z | hcs12z | s12z)
-		basic_machine=s12z-unknown
-		;;
-	m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*)
-		basic_machine=s12z-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	ms1)
-		basic_machine=mt-unknown
-		;;
-	strongarm | thumb | xscale)
-		basic_machine=arm-unknown
-		;;
-	xscaleeb)
-		basic_machine=armeb-unknown
-		;;
-
-	xscaleel)
-		basic_machine=armel-unknown
-		;;
-
-	# We use `pc' rather than `unknown'
-	# because (1) that's what they normally are, and
-	# (2) the word "unknown" tends to confuse beginning users.
-	i*86 | x86_64)
-	  basic_machine=$basic_machine-pc
-	  ;;
-	# Recognize the basic CPU types with company name.
-	1750a-* | 580-* \
-	| a29k-* \
-	| aarch64-* | aarch64_be-* \
-	| abacus-* \
-	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
-	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
-	| alphapca5[67]-* | alpha64pca5[67]-* \
-	| am33_2.0-* \
-	| arc-* | arceb-* \
-	| arm-*  | arm[lb]e-* | arme[lb]-* | armv*-* \
-	| avr-* | avr32-* \
-	| asmjs-* \
-	| ba-* \
-	| be32-* | be64-* \
-	| bfin-* | bs2000-* \
-	| c[123]* | c30-* | [cjt]90-* | c4x-* \
-	| c8051-* | clipper-* | craynv-* | csky-* | cydra-* \
-	| d10v-* | d30v-* | dlx-* | dsp16xx-* \
-	| e2k-* | elxsi-* | epiphany-* \
-	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | ft32-* | fx80-* \
-	| h8300-* | h8500-* \
-	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
-	| hexagon-* \
-	| i370-* | i*86-* | i860-* | i960-* | ia16-* | ia64-* \
-	| ip2k-* | iq2000-* \
-	| k1om-* \
-	| le32-* | le64-* \
-	| lm32-* \
-	| m32c-* | m32r-* | m32rle-* \
-	| m5200-* | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* | v70-* | w65-* \
-	| m6811-* | m68hc11-* | m6812-* | m68hc12-* | m68hcs12x-* | nvptx-* | picochip-* \
-	| m88110-* | m88k-* | maxq-* | mb-* | mcore-* | mep-* | metag-* \
-	| microblaze-* | microblazeel-* \
-	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
-	| mips16-* \
-	| mips64-* | mips64el-* \
-	| mips64octeon-* | mips64octeonel-* \
-	| mips64orion-* | mips64orionel-* \
-	| mips64r5900-* | mips64r5900el-* \
-	| mips64vr-* | mips64vrel-* \
-	| mips64vr4100-* | mips64vr4100el-* \
-	| mips64vr4300-* | mips64vr4300el-* \
-	| mips64vr5000-* | mips64vr5000el-* \
-	| mips64vr5900-* | mips64vr5900el-* \
-	| mipsisa32-* | mipsisa32el-* \
-	| mipsisa32r2-* | mipsisa32r2el-* \
-	| mipsisa32r6-* | mipsisa32r6el-* \
-	| mipsisa64-* | mipsisa64el-* \
-	| mipsisa64r2-* | mipsisa64r2el-* \
-	| mipsisa64r6-* | mipsisa64r6el-* \
-	| mipsisa64sb1-* | mipsisa64sb1el-* \
-	| mipsisa64sr71k-* | mipsisa64sr71kel-* \
-	| mipsr5900-* | mipsr5900el-* \
-	| mipstx39-* | mipstx39el-* \
-	| mmix-* \
-	| mn10200-* | mn10300-* \
-	| moxie-* \
-	| mt-* \
-	| msp430-* \
-	| nds32-* | nds32le-* | nds32be-* \
-	| nfp-* \
-	| nios-* | nios2-* | nios2eb-* | nios2el-* \
-	| none-* | np1-* | ns16k-* | ns32k-* \
-	| open8-* \
-	| or1k*-* \
-	| or32-* \
-	| orion-* \
-	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
-	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
-	| pru-* \
-	| pyramid-* \
-	| riscv-* | riscv32-* | riscv64-* \
-	| rl78-* | romp-* | rs6000-* | rx-* \
-	| score-* \
-	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]ae[lb]-* | sh[23]e-* | she[lb]-* | sh[lb]e-* \
-	| sh[1234]e[lb]-* |  sh[12345][lb]e-* | sh[23]ele-* | sh64-* | sh64le-* \
-	| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
-	| sparclite-* \
-	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
-	| spu-* \
-	| tahoe-* \
-	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
-	| tron-* \
-	| ubicom32-* \
-	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
-	| vax-* \
-	| visium-* \
-	| wasm32-* \
-	| we32k-* \
-	| x86-* | x86_64-* | xc16x-* | xgate-* | xps100-* \
-	| xstormy16-* | xtensa*-* \
-	| ymp-* \
-	| z8k-* | z80-*)
-		;;
-	# Recognize the basic CPU types without company name, with glob match.
-	xtensa*)
-		basic_machine=$basic_machine-unknown
-		;;
 	# Recognize the various machine names and aliases which stand
 	# for a CPU type and a company and sometimes even an OS.
 	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
-		basic_machine=m68000-att
+		cpu=m68000
+		vendor=att
 		;;
 	3b*)
-		basic_machine=we32k-att
-		;;
-	amd64)
-		basic_machine=x86_64-pc
-		;;
-	amd64-*)
-		basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	amiga | amiga-*)
-		basic_machine=m68k-unknown
-		;;
-	blackfin-*)
-		basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		os=linux
+		cpu=we32k
+		vendor=att
 		;;
 	bluegene*)
-		basic_machine=powerpc-ibm
-		os=cnk
-		;;
-	c54x-*)
-		basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	c55x-*)
-		basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	c6x-*)
-		basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	c90)
-		basic_machine=c90-cray
-		os=${os:-unicos}
-		;;
-	cr16 | cr16-*)
-		basic_machine=cr16-unknown
-		os=${os:-elf}
-		;;
-	crisv32 | crisv32-* | etraxfs*)
-		basic_machine=crisv32-axis
-		;;
-	cris | cris-* | etrax*)
-		basic_machine=cris-axis
-		;;
-	crx)
-		basic_machine=crx-unknown
-		os=${os:-elf}
-		;;
-	da30 | da30-*)
-		basic_machine=m68k-da30
-		;;
-	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
-		basic_machine=mips-dec
+		cpu=powerpc
+		vendor=ibm
+		basic_os=cnk
 		;;
 	decsystem10* | dec10*)
-		basic_machine=pdp10-dec
-		os=tops10
+		cpu=pdp10
+		vendor=dec
+		basic_os=tops10
 		;;
 	decsystem20* | dec20*)
-		basic_machine=pdp10-dec
-		os=tops20
+		cpu=pdp10
+		vendor=dec
+		basic_os=tops20
 		;;
 	delta | 3300 | motorola-3300 | motorola-delta \
 	      | 3300-motorola | delta-motorola)
-		basic_machine=m68k-motorola
-		;;
-	dpx20 | dpx20-*)
-		basic_machine=rs6000-bull
-		os=${os:-bosx}
+		cpu=m68k
+		vendor=motorola
 		;;
 	dpx2*)
-		basic_machine=m68k-bull
-		os=sysv3
-		;;
-	e500v[12])
-		basic_machine=powerpc-unknown
-		os=$os"spe"
-		;;
-	e500v[12]-*)
-		basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		os=$os"spe"
+		cpu=m68k
+		vendor=bull
+		basic_os=sysv3
 		;;
 	encore | umax | mmax)
-		basic_machine=ns32k-encore
+		cpu=ns32k
+		vendor=encore
 		;;
 	elxsi)
-		basic_machine=elxsi-elxsi
-		os=${os:-bsd}
+		cpu=elxsi
+		vendor=elxsi
+		basic_os=${basic_os:-bsd}
 		;;
 	fx2800)
-		basic_machine=i860-alliant
+		cpu=i860
+		vendor=alliant
 		;;
 	genix)
-		basic_machine=ns32k-ns
+		cpu=ns32k
+		vendor=ns
 		;;
 	h3050r* | hiux*)
-		basic_machine=hppa1.1-hitachi
-		os=hiuxwe2
-		;;
-	hp300-*)
-		basic_machine=m68k-hp
+		cpu=hppa1.1
+		vendor=hitachi
+		basic_os=hiuxwe2
 		;;
 	hp3k9[0-9][0-9] | hp9[0-9][0-9])
-		basic_machine=hppa1.0-hp
+		cpu=hppa1.0
+		vendor=hp
 		;;
 	hp9k2[0-9][0-9] | hp9k31[0-9])
-		basic_machine=m68000-hp
+		cpu=m68000
+		vendor=hp
 		;;
 	hp9k3[2-9][0-9])
-		basic_machine=m68k-hp
+		cpu=m68k
+		vendor=hp
 		;;
 	hp9k6[0-9][0-9] | hp6[0-9][0-9])
-		basic_machine=hppa1.0-hp
+		cpu=hppa1.0
+		vendor=hp
 		;;
 	hp9k7[0-79][0-9] | hp7[0-79][0-9])
-		basic_machine=hppa1.1-hp
+		cpu=hppa1.1
+		vendor=hp
 		;;
 	hp9k78[0-9] | hp78[0-9])
 		# FIXME: really hppa2.0-hp
-		basic_machine=hppa1.1-hp
+		cpu=hppa1.1
+		vendor=hp
 		;;
 	hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893)
 		# FIXME: really hppa2.0-hp
-		basic_machine=hppa1.1-hp
+		cpu=hppa1.1
+		vendor=hp
 		;;
 	hp9k8[0-9][13679] | hp8[0-9][13679])
-		basic_machine=hppa1.1-hp
+		cpu=hppa1.1
+		vendor=hp
 		;;
 	hp9k8[0-9][0-9] | hp8[0-9][0-9])
-		basic_machine=hppa1.0-hp
+		cpu=hppa1.0
+		vendor=hp
 		;;
 	i*86v32)
-		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
-		os=sysv32
+		cpu=$(echo "$1" | sed -e 's/86.*/86/')
+		vendor=pc
+		basic_os=sysv32
 		;;
 	i*86v4*)
-		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
-		os=sysv4
+		cpu=$(echo "$1" | sed -e 's/86.*/86/')
+		vendor=pc
+		basic_os=sysv4
 		;;
 	i*86v)
-		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
-		os=sysv
+		cpu=$(echo "$1" | sed -e 's/86.*/86/')
+		vendor=pc
+		basic_os=sysv
 		;;
 	i*86sol2)
-		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
-		os=solaris2
+		cpu=$(echo "$1" | sed -e 's/86.*/86/')
+		vendor=pc
+		basic_os=solaris2
 		;;
 	j90 | j90-cray)
-		basic_machine=j90-cray
-		os=${os:-unicos}
+		cpu=j90
+		vendor=cray
+		basic_os=${basic_os:-unicos}
 		;;
 	iris | iris4d)
-		basic_machine=mips-sgi
-		case $os in
+		cpu=mips
+		vendor=sgi
+		case $basic_os in
 		    irix*)
 			;;
 		    *)
-			os=irix4
+			basic_os=irix4
 			;;
 		esac
 		;;
-	leon-*|leon[3-9]-*)
-		basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'`
-		;;
-	m68knommu-*)
-		basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		os=linux
-		;;
-	microblaze*)
-		basic_machine=microblaze-xilinx
-		;;
 	miniframe)
-		basic_machine=m68000-convergent
+		cpu=m68000
+		vendor=convergent
 		;;
 	*mint | mint[0-9]* | *MiNT | *MiNT[0-9]*)
-		basic_machine=m68k-atari
-		os=mint
-		;;
-	mips3*-*)
-		basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`
-		;;
-	mips3*)
-		basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown
-		;;
-	ms1-*)
-		basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'`
+		cpu=m68k
+		vendor=atari
+		basic_os=mint
 		;;
 	news-3600 | risc-news)
-		basic_machine=mips-sony
-		os=newsos
+		cpu=mips
+		vendor=sony
+		basic_os=newsos
 		;;
 	next | m*-next)
-		basic_machine=m68k-next
-		case $os in
-		    nextstep* )
+		cpu=m68k
+		vendor=next
+		case $basic_os in
+		    openstep*)
+		        ;;
+		    nextstep*)
 			;;
 		    ns2*)
-		      os=nextstep2
+		      basic_os=nextstep2
 			;;
 		    *)
-		      os=nextstep3
+		      basic_os=nextstep3
 			;;
 		esac
 		;;
 	np1)
-		basic_machine=np1-gould
-		;;
-	neo-tandem)
-		basic_machine=neo-tandem
-		;;
-	nse-tandem)
-		basic_machine=nse-tandem
-		;;
-	nsr-tandem)
-		basic_machine=nsr-tandem
-		;;
-	nsv-tandem)
-		basic_machine=nsv-tandem
-		;;
-	nsx-tandem)
-		basic_machine=nsx-tandem
+		cpu=np1
+		vendor=gould
 		;;
 	op50n-* | op60c-*)
-		basic_machine=hppa1.1-oki
-		os=proelf
-		;;
-	openrisc | openrisc-*)
-		basic_machine=or32-unknown
+		cpu=hppa1.1
+		vendor=oki
+		basic_os=proelf
 		;;
 	pa-hitachi)
-		basic_machine=hppa1.1-hitachi
-		os=hiuxwe2
-		;;
-	parisc-*)
-		basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		os=linux
+		cpu=hppa1.1
+		vendor=hitachi
+		basic_os=hiuxwe2
 		;;
 	pbd)
-		basic_machine=sparc-tti
+		cpu=sparc
+		vendor=tti
 		;;
 	pbb)
-		basic_machine=m68k-tti
+		cpu=m68k
+		vendor=tti
 		;;
-	pc532 | pc532-*)
-		basic_machine=ns32k-pc532
-		;;
-	pc98)
-		basic_machine=i386-pc
-		;;
-	pc98-*)
-		basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	pentium | p5 | k5 | k6 | nexgen | viac3)
-		basic_machine=i586-pc
-		;;
-	pentiumpro | p6 | 6x86 | athlon | athlon_*)
-		basic_machine=i686-pc
-		;;
-	pentiumii | pentium2 | pentiumiii | pentium3)
-		basic_machine=i686-pc
-		;;
-	pentium4)
-		basic_machine=i786-pc
-		;;
-	pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*)
-		basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	pentiumpro-* | p6-* | 6x86-* | athlon-*)
-		basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*)
-		basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	pentium4-*)
-		basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+	pc532)
+		cpu=ns32k
+		vendor=pc532
 		;;
 	pn)
-		basic_machine=pn-gould
+		cpu=pn
+		vendor=gould
 		;;
-	power)	basic_machine=power-ibm
-		;;
-	ppc | ppcbe)	basic_machine=powerpc-unknown
-		;;
-	ppc-* | ppcbe-*)
-		basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	ppcle | powerpclittle)
-		basic_machine=powerpcle-unknown
-		;;
-	ppcle-* | powerpclittle-*)
-		basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	ppc64)	basic_machine=powerpc64-unknown
-		;;
-	ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	ppc64le | powerpc64little)
-		basic_machine=powerpc64le-unknown
-		;;
-	ppc64le-* | powerpc64little-*)
-		basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+	power)
+		cpu=power
+		vendor=ibm
 		;;
 	ps2)
-		basic_machine=i386-ibm
+		cpu=i386
+		vendor=ibm
 		;;
 	rm[46]00)
-		basic_machine=mips-siemens
+		cpu=mips
+		vendor=siemens
 		;;
 	rtpc | rtpc-*)
-		basic_machine=romp-ibm
-		;;
-	s390 | s390-*)
-		basic_machine=s390-ibm
-		;;
-	s390x | s390x-*)
-		basic_machine=s390x-ibm
-		;;
-	sb1)
-		basic_machine=mipsisa64sb1-unknown
-		;;
-	sb1el)
-		basic_machine=mipsisa64sb1el-unknown
+		cpu=romp
+		vendor=ibm
 		;;
 	sde)
-		basic_machine=mipsisa32-sde
-		os=${os:-elf}
-		;;
-	sequent)
-		basic_machine=i386-sequent
-		;;
-	sh5el)
-		basic_machine=sh5le-unknown
-		;;
-	sh5el-*)
-		basic_machine=sh5le-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+		cpu=mipsisa32
+		vendor=sde
+		basic_os=${basic_os:-elf}
 		;;
 	simso-wrs)
-		basic_machine=sparclite-wrs
-		os=vxworks
-		;;
-	spur)
-		basic_machine=spur-unknown
-		;;
-	strongarm-* | thumb-*)
-		basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'`
-		;;
-	tile*-*)
-		;;
-	tile*)
-		basic_machine=$basic_machine-unknown
-		os=${os:-linux-gnu}
-		;;
-	tx39)
-		basic_machine=mipstx39-unknown
-		;;
-	tx39el)
-		basic_machine=mipstx39el-unknown
+		cpu=sparclite
+		vendor=wrs
+		basic_os=vxworks
 		;;
 	tower | tower-32)
-		basic_machine=m68k-ncr
+		cpu=m68k
+		vendor=ncr
 		;;
 	vpp*|vx|vx-*)
-		basic_machine=f301-fujitsu
+		cpu=f301
+		vendor=fujitsu
 		;;
-	w65*)
-		basic_machine=w65-wdc
-		os=none
+	w65)
+		cpu=w65
+		vendor=wdc
 		;;
 	w89k-*)
-		basic_machine=hppa1.1-winbond
-		os=proelf
-		;;
-	x64)
-		basic_machine=x86_64-pc
-		;;
-	xscale-* | xscalee[bl]-*)
-		basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'`
+		cpu=hppa1.1
+		vendor=winbond
+		basic_os=proelf
 		;;
 	none)
-		basic_machine=none-none
+		cpu=none
+		vendor=none
+		;;
+	leon|leon[3-9])
+		cpu=sparc
+		vendor=$basic_machine
+		;;
+	leon-*|leon[3-9]-*)
+		cpu=sparc
+		vendor=$(echo "$basic_machine" | sed 's/-.*//')
+		;;
+
+	*-*)
+		# shellcheck disable=SC2162
+		IFS="-" read cpu vendor <<EOF
+$basic_machine
+EOF
+		;;
+	# We use `pc' rather than `unknown'
+	# because (1) that's what they normally are, and
+	# (2) the word "unknown" tends to confuse beginning users.
+	i*86 | x86_64)
+		cpu=$basic_machine
+		vendor=pc
+		;;
+	# These rules are duplicated from below for sake of the special case above;
+	# i.e. things that normalized to x86 arches should also default to "pc"
+	pc98)
+		cpu=i386
+		vendor=pc
+		;;
+	x64 | amd64)
+		cpu=x86_64
+		vendor=pc
+		;;
+	# Recognize the basic CPU types without company name.
+	*)
+		cpu=$basic_machine
+		vendor=unknown
+		;;
+esac
+
+unset -v basic_machine
+
+# Decode basic machines in the full and proper CPU-Company form.
+case $cpu-$vendor in
+	# Here we handle the default manufacturer of certain CPU types in canonical form. It is in
+	# some cases the only manufacturer, in others, it is the most popular.
+	craynv-unknown)
+		vendor=cray
+		basic_os=${basic_os:-unicosmp}
+		;;
+	c90-unknown | c90-cray)
+		vendor=cray
+		basic_os=${Basic_os:-unicos}
+		;;
+	fx80-unknown)
+		vendor=alliant
+		;;
+	romp-unknown)
+		vendor=ibm
+		;;
+	mmix-unknown)
+		vendor=knuth
+		;;
+	microblaze-unknown | microblazeel-unknown)
+		vendor=xilinx
+		;;
+	rs6000-unknown)
+		vendor=ibm
+		;;
+	vax-unknown)
+		vendor=dec
+		;;
+	pdp11-unknown)
+		vendor=dec
+		;;
+	we32k-unknown)
+		vendor=att
+		;;
+	cydra-unknown)
+		vendor=cydrome
+		;;
+	i370-ibm*)
+		vendor=ibm
+		;;
+	orion-unknown)
+		vendor=highlevel
+		;;
+	xps-unknown | xps100-unknown)
+		cpu=xps100
+		vendor=honeywell
+		;;
+
+	# Here we normalize CPU types with a missing or matching vendor
+	dpx20-unknown | dpx20-bull)
+		cpu=rs6000
+		vendor=bull
+		basic_os=${basic_os:-bosx}
+		;;
+
+	# Here we normalize CPU types irrespective of the vendor
+	amd64-*)
+		cpu=x86_64
+		;;
+	blackfin-*)
+		cpu=bfin
+		basic_os=linux
+		;;
+	c54x-*)
+		cpu=tic54x
+		;;
+	c55x-*)
+		cpu=tic55x
+		;;
+	c6x-*)
+		cpu=tic6x
+		;;
+	e500v[12]-*)
+		cpu=powerpc
+		basic_os=${basic_os}"spe"
+		;;
+	mips3*-*)
+		cpu=mips64
+		;;
+	ms1-*)
+		cpu=mt
+		;;
+	m68knommu-*)
+		cpu=m68k
+		basic_os=linux
+		;;
+	m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*)
+		cpu=s12z
+		;;
+	openrisc-*)
+		cpu=or32
+		;;
+	parisc-*)
+		cpu=hppa
+		basic_os=linux
+		;;
+	pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*)
+		cpu=i586
+		;;
+	pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*)
+		cpu=i686
+		;;
+	pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*)
+		cpu=i686
+		;;
+	pentium4-*)
+		cpu=i786
+		;;
+	pc98-*)
+		cpu=i386
+		;;
+	ppc-* | ppcbe-*)
+		cpu=powerpc
+		;;
+	ppcle-* | powerpclittle-*)
+		cpu=powerpcle
+		;;
+	ppc64-*)
+		cpu=powerpc64
+		;;
+	ppc64le-* | powerpc64little-*)
+		cpu=powerpc64le
+		;;
+	sb1-*)
+		cpu=mipsisa64sb1
+		;;
+	sb1el-*)
+		cpu=mipsisa64sb1el
+		;;
+	sh5e[lb]-*)
+		cpu=$(echo "$cpu" | sed 's/^\(sh.\)e\(.\)$/\1\2e/')
+		;;
+	spur-*)
+		cpu=spur
+		;;
+	strongarm-* | thumb-*)
+		cpu=arm
+		;;
+	tx39-*)
+		cpu=mipstx39
+		;;
+	tx39el-*)
+		cpu=mipstx39el
+		;;
+	x64-*)
+		cpu=x86_64
+		;;
+	xscale-* | xscalee[bl]-*)
+		cpu=$(echo "$cpu" | sed 's/^xscale/arm/')
+		;;
+	arm64-*)
+		cpu=aarch64
+		;;
+
+	# Recognize the canonical CPU Types that limit and/or modify the
+	# company names they are paired with.
+	cr16-*)
+		basic_os=${basic_os:-elf}
+		;;
+	crisv32-* | etraxfs*-*)
+		cpu=crisv32
+		vendor=axis
+		;;
+	cris-* | etrax*-*)
+		cpu=cris
+		vendor=axis
+		;;
+	crx-*)
+		basic_os=${basic_os:-elf}
+		;;
+	neo-tandem)
+		cpu=neo
+		vendor=tandem
+		;;
+	nse-tandem)
+		cpu=nse
+		vendor=tandem
+		;;
+	nsr-tandem)
+		cpu=nsr
+		vendor=tandem
+		;;
+	nsv-tandem)
+		cpu=nsv
+		vendor=tandem
+		;;
+	nsx-tandem)
+		cpu=nsx
+		vendor=tandem
+		;;
+	mipsallegrexel-sony)
+		cpu=mipsallegrexel
+		vendor=sony
+		;;
+	tile*-*)
+		basic_os=${basic_os:-linux-gnu}
 		;;
 
 	*)
-		echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2
-		exit 1
+		# Recognize the canonical CPU types that are allowed with any
+		# company name.
+		case $cpu in
+			1750a | 580 \
+			| a29k \
+			| aarch64 | aarch64_be \
+			| abacus \
+			| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \
+			| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \
+			| alphapca5[67] | alpha64pca5[67] \
+			| am33_2.0 \
+			| amdgcn \
+			| arc | arceb \
+			| arm | arm[lb]e | arme[lb] | armv* \
+			| avr | avr32 \
+			| asmjs \
+			| ba \
+			| be32 | be64 \
+			| bfin | bpf | bs2000 \
+			| c[123]* | c30 | [cjt]90 | c4x \
+			| c8051 | clipper | craynv | csky | cydra \
+			| d10v | d30v | dlx | dsp16xx \
+			| e2k | elxsi | epiphany \
+			| f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \
+			| h8300 | h8500 \
+			| hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
+			| hexagon \
+			| i370 | i*86 | i860 | i960 | ia16 | ia64 \
+			| ip2k | iq2000 \
+			| k1om \
+			| le32 | le64 \
+			| lm32 \
+			| m32c | m32r | m32rle \
+			| m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \
+			| m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \
+			| m88110 | m88k | maxq | mb | mcore | mep | metag \
+			| microblaze | microblazeel \
+			| mips | mipsbe | mipseb | mipsel | mipsle \
+			| mips16 \
+			| mips64 | mips64eb | mips64el \
+			| mips64octeon | mips64octeonel \
+			| mips64orion | mips64orionel \
+			| mips64r5900 | mips64r5900el \
+			| mips64vr | mips64vrel \
+			| mips64vr4100 | mips64vr4100el \
+			| mips64vr4300 | mips64vr4300el \
+			| mips64vr5000 | mips64vr5000el \
+			| mips64vr5900 | mips64vr5900el \
+			| mipsisa32 | mipsisa32el \
+			| mipsisa32r2 | mipsisa32r2el \
+			| mipsisa32r6 | mipsisa32r6el \
+			| mipsisa64 | mipsisa64el \
+			| mipsisa64r2 | mipsisa64r2el \
+			| mipsisa64r6 | mipsisa64r6el \
+			| mipsisa64sb1 | mipsisa64sb1el \
+			| mipsisa64sr71k | mipsisa64sr71kel \
+			| mipsr5900 | mipsr5900el \
+			| mipstx39 | mipstx39el \
+			| mmix \
+			| mn10200 | mn10300 \
+			| moxie \
+			| mt \
+			| msp430 \
+			| nds32 | nds32le | nds32be \
+			| nfp \
+			| nios | nios2 | nios2eb | nios2el \
+			| none | np1 | ns16k | ns32k | nvptx \
+			| open8 \
+			| or1k* \
+			| or32 \
+			| orion \
+			| picochip \
+			| pdp10 | pdp11 | pj | pjl | pn | power \
+			| powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \
+			| pru \
+			| pyramid \
+			| riscv | riscv32 | riscv64 \
+			| rl78 | romp | rs6000 | rx \
+			| s390 | s390x \
+			| score \
+			| sh | shl \
+			| sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \
+			| sh[1234]e[lb] |  sh[12345][lb]e | sh[23]ele | sh64 | sh64le \
+			| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \
+			| sparclite \
+			| sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \
+			| spu \
+			| tahoe \
+			| thumbv7* \
+			| tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \
+			| tron \
+			| ubicom32 \
+			| v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \
+			| vax \
+			| visium \
+			| w65 \
+			| wasm32 | wasm64 \
+			| we32k \
+			| x86 | x86_64 | xc16x | xgate | xps100 \
+			| xstormy16 | xtensa* \
+			| ymp \
+			| z8k | z80)
+				;;
+
+			*)
+				echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2
+				exit 1
+				;;
+		esac
 		;;
 esac
 
 # Here we canonicalize certain aliases for manufacturers.
-case $basic_machine in
-	*-digital*)
-		basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'`
+case $vendor in
+	digital*)
+		vendor=dec
 		;;
-	*-commodore*)
-		basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'`
+	commodore*)
+		vendor=cbm
 		;;
 	*)
 		;;
@@ -1296,8 +1279,47 @@ esac
 
 # Decode manufacturer-specific aliases for certain operating systems.
 
-if [ x$os != x ]
+if test x$basic_os != x
 then
+
+# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just
+# set os.
+case $basic_os in
+	gnu/linux*)
+		kernel=linux
+		os=$(echo $basic_os | sed -e 's|gnu/linux|gnu|')
+		;;
+	os2-emx)
+		kernel=os2
+		os=$(echo $basic_os | sed -e 's|os2-emx|emx|')
+		;;
+	nto-qnx*)
+		kernel=nto
+		os=$(echo $basic_os | sed -e 's|nto-qnx|qnx|')
+		;;
+	*-*)
+		# shellcheck disable=SC2162
+		IFS="-" read kernel os <<EOF
+$basic_os
+EOF
+		;;
+	# Default OS when just kernel was specified
+	nto*)
+		kernel=nto
+		os=$(echo $basic_os | sed -e 's|nto|qnx|')
+		;;
+	linux*)
+		kernel=linux
+		os=$(echo $basic_os | sed -e 's|linux|gnu|')
+		;;
+	*)
+		kernel=
+		os=$basic_os
+		;;
+esac
+
+# Now, normalize the OS (knowing we just have one component, it's not a kernel,
+# etc.)
 case $os in
 	# First match some system type aliases that might get confused
 	# with valid system types.
@@ -1309,7 +1331,7 @@ case $os in
 		os=cnk
 		;;
 	solaris1 | solaris1.*)
-		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		os=$(echo $os | sed -e 's|solaris1|sunos4|')
 		;;
 	solaris)
 		os=solaris2
@@ -1317,9 +1339,6 @@ case $os in
 	unixware*)
 		os=sysv4.2uw
 		;;
-	gnu/linux*)
-		os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'`
-		;;
 	# es1800 is here to avoid being matched by es* (a different OS)
 	es1800*)
 		os=ose
@@ -1341,12 +1360,9 @@ case $os in
 		os=sco3.2v4
 		;;
 	sco3.2.[4-9]*)
-		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		os=$(echo $os | sed -e 's/sco3.2./sco3.2v/')
 		;;
-	sco3.2v[4-9]* | sco5v6*)
-		# Don't forget version if it is 3.2v4 or newer.
-		;;
-	scout)
+	sco*v* | scout)
 		# Don't match below
 		;;
 	sco*)
@@ -1355,78 +1371,26 @@ case $os in
 	psos*)
 		os=psos
 		;;
-	# Now accept the basic system types.
-	# The portable systems comes first.
-	# Each alternative MUST end in a * to match a version number.
-	# sysv* is not here because it comes later, after sysvr4.
-	gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \
-	     | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\
-	     | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \
-	     | sym* | kopensolaris* | plan9* \
-	     | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \
-	     | aos* | aros* | cloudabi* | sortix* \
-	     | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \
-	     | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \
-	     | knetbsd* | mirbsd* | netbsd* \
-	     | bitrig* | openbsd* | solidbsd* | libertybsd* \
-	     | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \
-	     | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \
-	     | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \
-	     | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \
-	     | chorusrdb* | cegcc* | glidix* \
-	     | cygwin* | msys* | pe* | moss* | proelf* | rtems* \
-	     | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \
-	     | linux-newlib* | linux-musl* | linux-uclibc* \
-	     | uxpv* | beos* | mpeix* | udk* | moxiebox* \
-	     | interix* | uwin* | mks* | rhapsody* | darwin* \
-	     | openstep* | oskit* | conix* | pw32* | nonstopux* \
-	     | storm-chaos* | tops10* | tenex* | tops20* | its* \
-	     | os2* | vos* | palmos* | uclinux* | nucleus* \
-	     | morphos* | superux* | rtmk* | windiss* \
-	     | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \
-	     | skyos* | haiku* | rdos* | toppers* | drops* | es* \
-	     | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \
-	     | midnightbsd*)
-	# Remember, each alternative MUST END IN *, to match a version number.
-		;;
 	qnx*)
-		case $basic_machine in
-		    x86-* | i*86-*)
-			;;
-		    *)
-			os=nto-$os
-			;;
-		esac
+		os=qnx
 		;;
 	hiux*)
 		os=hiuxwe2
 		;;
-	nto-qnx*)
-		;;
-	nto*)
-		os=`echo $os | sed -e 's|nto|nto-qnx|'`
-		;;
-	sim | xray | os68k* | v88r* \
-	    | windows* | osx | abug | netware* | os9* \
-	    | macos* | mpw* | magic* | mmixware* | mon960* | lnews*)
-		;;
-	linux-dietlibc)
-		os=linux-dietlibc
-		;;
-	linux*)
-		os=`echo $os | sed -e 's|linux|linux-gnu|'`
-		;;
 	lynx*178)
 		os=lynxos178
 		;;
 	lynx*5)
 		os=lynxos5
 		;;
+	lynxos*)
+		# don't get caught up in next wildcard
+		;;
 	lynx*)
 		os=lynxos
 		;;
-	mac*)
-		os=`echo "$os" | sed -e 's|mac|macos|'`
+	mac[0-9]*)
+		os=$(echo "$os" | sed -e 's|mac|macos|')
 		;;
 	opened*)
 		os=openedition
@@ -1435,10 +1399,10 @@ case $os in
 		os=os400
 		;;
 	sunos5*)
-		os=`echo "$os" | sed -e 's|sunos5|solaris2|'`
+		os=$(echo "$os" | sed -e 's|sunos5|solaris2|')
 		;;
 	sunos6*)
-		os=`echo "$os" | sed -e 's|sunos6|solaris3|'`
+		os=$(echo "$os" | sed -e 's|sunos6|solaris3|')
 		;;
 	wince*)
 		os=wince
@@ -1470,12 +1434,9 @@ case $os in
 	ns2)
 		os=nextstep2
 		;;
-	nsk*)
-		os=nsk
-		;;
 	# Preserve the version number of sinix5.
 	sinix5.*)
-		os=`echo $os | sed -e 's|sinix|sysv|'`
+		os=$(echo $os | sed -e 's|sinix|sysv|')
 		;;
 	sinix*)
 		os=sysv4
@@ -1498,18 +1459,12 @@ case $os in
 	sysvr4)
 		os=sysv4
 		;;
-	# This must come after sysvr4.
-	sysv*)
-		;;
 	ose*)
 		os=ose
 		;;
 	*mint | mint[0-9]* | *MiNT | MiNT[0-9]*)
 		os=mint
 		;;
-	zvmoe)
-		os=zvmoe
-		;;
 	dicos*)
 		os=dicos
 		;;
@@ -1517,7 +1472,7 @@ case $os in
 		# Until real need of OS specific support for
 		# particular features comes up, bare metal
 		# configurations are quite functional.
-		case $basic_machine in
+		case $cpu in
 		    arm*)
 			os=eabi
 			;;
@@ -1526,19 +1481,11 @@ case $os in
 			;;
 		esac
 		;;
-	nacl*)
-		;;
-	ios)
-		;;
-	none)
-		;;
-	*-eabi)
-		;;
 	*)
-		echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2
-		exit 1
+		# No normalization, but not necessarily accepted, that comes below.
 		;;
 esac
+
 else
 
 # Here we handle the default operating systems that come with various machines.
@@ -1551,7 +1498,8 @@ else
 # will signal an error saying that MANUFACTURER isn't an operating
 # system, and we'll never get to this point.
 
-case $basic_machine in
+kernel=
+case $cpu-$vendor in
 	score-*)
 		os=elf
 		;;
@@ -1562,7 +1510,8 @@ case $basic_machine in
 		os=riscix1.2
 		;;
 	arm*-rebel)
-		os=linux
+		kernel=linux
+		os=gnu
 		;;
 	arm*-semi)
 		os=aout
@@ -1728,86 +1677,173 @@ case $basic_machine in
 		os=none
 		;;
 esac
+
 fi
 
-# Here we handle the case where we know the os, and the CPU type, but not the
-# manufacturer.  We pick the logical manufacturer.
-vendor=unknown
-case $basic_machine in
-	*-unknown)
-		case $os in
-			riscix*)
-				vendor=acorn
-				;;
-			sunos*)
-				vendor=sun
-				;;
-			cnk*|-aix*)
-				vendor=ibm
-				;;
-			beos*)
-				vendor=be
-				;;
-			hpux*)
-				vendor=hp
-				;;
-			mpeix*)
-				vendor=hp
-				;;
-			hiux*)
-				vendor=hitachi
-				;;
-			unos*)
-				vendor=crds
-				;;
-			dgux*)
-				vendor=dg
-				;;
-			luna*)
-				vendor=omron
-				;;
-			genix*)
-				vendor=ns
-				;;
-			clix*)
-				vendor=intergraph
-				;;
-			mvs* | opened*)
-				vendor=ibm
-				;;
-			os400*)
-				vendor=ibm
-				;;
-			ptx*)
-				vendor=sequent
-				;;
-			tpf*)
-				vendor=ibm
-				;;
-			vxsim* | vxworks* | windiss*)
-				vendor=wrs
-				;;
-			aux*)
-				vendor=apple
-				;;
-			hms*)
-				vendor=hitachi
-				;;
-			mpw* | macos*)
-				vendor=apple
-				;;
-			*mint | mint[0-9]* | *MiNT | MiNT[0-9]*)
-				vendor=atari
-				;;
-			vos*)
-				vendor=stratus
-				;;
-		esac
-		basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"`
+# Now, validate our (potentially fixed-up) OS.
+case $os in
+	# Sometimes we do "kernel-abi", so those need to count as OSes.
+	musl* | newlib* | uclibc*)
+		;;
+	# Likewise for "kernel-libc"
+	eabi | eabihf | gnueabi | gnueabihf)
+		;;
+	# Now accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative MUST end in a * to match a version number.
+	gnu* | android* | bsd* | mach* | minix* | genix* | ultrix* | irix* \
+	     | *vms* | esix* | aix* | cnk* | sunos | sunos[34]* \
+	     | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \
+	     | sym* |  plan9* | psp* | sim* | xray* | os68k* | v88r* \
+	     | hiux* | abug | nacl* | netware* | windows* \
+	     | os9* | macos* | osx* | ios* \
+	     | mpw* | magic* | mmixware* | mon960* | lnews* \
+	     | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \
+	     | aos* | aros* | cloudabi* | sortix* | twizzler* \
+	     | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \
+	     | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \
+	     | mirbsd* | netbsd* | dicos* | openedition* | ose* \
+	     | bitrig* | openbsd* | solidbsd* | libertybsd* | os108* \
+	     | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \
+	     | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \
+	     | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \
+	     | udi* | lites* | ieee* | go32* | aux* | hcos* \
+	     | chorusrdb* | cegcc* | glidix* \
+	     | cygwin* | msys* | pe* | moss* | proelf* | rtems* \
+	     | midipix* | mingw32* | mingw64* | mint* \
+	     | uxpv* | beos* | mpeix* | udk* | moxiebox* \
+	     | interix* | uwin* | mks* | rhapsody* | darwin* \
+	     | openstep* | oskit* | conix* | pw32* | nonstopux* \
+	     | storm-chaos* | tops10* | tenex* | tops20* | its* \
+	     | os2* | vos* | palmos* | uclinux* | nucleus* | morphos* \
+	     | scout* | superux* | sysv* | rtmk* | tpf* | windiss* \
+	     | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \
+	     | skyos* | haiku* | rdos* | toppers* | drops* | es* \
+	     | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \
+	     | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \
+	     | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx*)
+		;;
+	# This one is extra strict with allowed versions
+	sco3.2v2 | sco3.2v[4-9]* | sco5v6*)
+		# Don't forget version if it is 3.2v4 or newer.
+		;;
+	none)
+		;;
+	*)
+		echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2
+		exit 1
 		;;
 esac
 
-echo "$basic_machine-$os"
+# As a final step for OS-related things, validate the OS-kernel combination
+# (given a valid OS), if there is a kernel.
+case $kernel-$os in
+	linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* )
+		;;
+	uclinux-uclibc* )
+		;;
+	-dietlibc* | -newlib* | -musl* | -uclibc* )
+		# These are just libc implementations, not actual OSes, and thus
+		# require a kernel.
+		echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2
+		exit 1
+		;;
+	kfreebsd*-gnu* | kopensolaris*-gnu*)
+		;;
+	nto-qnx*)
+		;;
+	os2-emx)
+		;;
+	*-eabi* | *-gnueabi*)
+		;;
+	-*)
+		# Blank kernel with real OS is always fine.
+		;;
+	*-*)
+		echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2
+		exit 1
+		;;
+esac
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+case $vendor in
+	unknown)
+		case $cpu-$os in
+			*-riscix*)
+				vendor=acorn
+				;;
+			*-sunos*)
+				vendor=sun
+				;;
+			*-cnk* | *-aix*)
+				vendor=ibm
+				;;
+			*-beos*)
+				vendor=be
+				;;
+			*-hpux*)
+				vendor=hp
+				;;
+			*-mpeix*)
+				vendor=hp
+				;;
+			*-hiux*)
+				vendor=hitachi
+				;;
+			*-unos*)
+				vendor=crds
+				;;
+			*-dgux*)
+				vendor=dg
+				;;
+			*-luna*)
+				vendor=omron
+				;;
+			*-genix*)
+				vendor=ns
+				;;
+			*-clix*)
+				vendor=intergraph
+				;;
+			*-mvs* | *-opened*)
+				vendor=ibm
+				;;
+			*-os400*)
+				vendor=ibm
+				;;
+			s390-* | s390x-*)
+				vendor=ibm
+				;;
+			*-ptx*)
+				vendor=sequent
+				;;
+			*-tpf*)
+				vendor=ibm
+				;;
+			*-vxsim* | *-vxworks* | *-windiss*)
+				vendor=wrs
+				;;
+			*-aux*)
+				vendor=apple
+				;;
+			*-hms*)
+				vendor=hitachi
+				;;
+			*-mpw* | *-macos*)
+				vendor=apple
+				;;
+			*-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*)
+				vendor=atari
+				;;
+			*-vos*)
+				vendor=stratus
+				;;
+		esac
+		;;
+esac
+
+echo "$cpu-$vendor-${kernel:+$kernel-}$os"
 exit
 
 # Local variables:

From 0afab668fa3f20f091dec0a31fc0b0fbaac2afde Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 3 Dec 2020 13:19:20 +0100
Subject: [PATCH 525/882] Don't fail early when -j0 is passed

If the build closure contains some CA derivations, then we can't know
ahead-of-time that we won't build anything as early-cutoff might come-in
at a laster stage
---
 src/libstore/build/worker.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 1f8999a4b..6c96a93bd 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -223,11 +223,6 @@ void Worker::run(const Goals & _topGoals)
     uint64_t downloadSize, narSize;
     store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
-    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
-        throw Error(
-            "%d derivations need to be built, but neither local builds ('--max-jobs') "
-            "nor remote builds ('--builders') are enabled", willBuild.size());
-
     debug("entered goal loop");
 
     while (1) {

From 8ad72b1f1c272d01d5d67b21553fc261c8df5302 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 3 Dec 2020 13:20:53 +0100
Subject: [PATCH 526/882] Properly test early cutoff with CA derivations

Build things with a different seed each time to make sure that it works
despite the different drvs
---
 tests/content-addressed.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 52f7529b5..03eff549c 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -7,30 +7,32 @@ nix --experimental-features 'nix-command ca-derivations' show-derivation --deriv
 
 buildAttr () {
     local derivationPath=$1
-    shift
-    local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
+    local seedValue=$2
+    shift; shift
+    local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link")
     args+=("$@")
     nix-build "${args[@]}"
 }
 
 testRemoteCache () {
     clearCache
-    local outPath=$(buildAttr dependentNonCA)
+    local outPath=$(buildAttr dependentNonCA 1)
     nix copy --to file://$cacheDir $outPath
     clearStore
-    buildAttr dependentNonCA --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca")
+    buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca")
 }
 
 testDeterministicCA () {
-    [[ $(buildAttr rootCA) = $(buildAttr rootCA) ]]
+    [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]]
 }
 
 testCutoffFor () {
     local out1 out2
-    out1=$(buildAttr $1)
+    out1=$(buildAttr $1 1)
     # The seed only changes the root derivation, and not it's output, so the
     # dependent derivations should only need to be built once.
-    out2=$(buildAttr $1 -j0)
+    buildAttr rootCA 2
+    out2=$(buildAttr $1 2 -j0)
     test "$out1" == "$out2"
 }
 

From c3c858ac6d0c75bd95bd6913276ef20cf2495e96 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 13:55:19 +0100
Subject: [PATCH 527/882] Make doc() return arbitrary Markdown rather than the
 contents of the "Description" section

Thus we can return the examples section (and any other sections) from
doc() and don't need examples() anymore.
---
 doc/manual/generate-manpage.nix |  2 +-
 src/nix/add-to-store.cc         | 26 ++++++++++++++++++++------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 4a0b0290b..fbd7f3e7d 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -26,7 +26,7 @@ let
          + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
        else "")
     + (if def ? doc
-       then "# Description\n\n" + def.doc + "\n\n"
+       then def.doc + "\n\n"
        else "")
     + (let s = showFlags def.flags; in
        if s != ""
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index df51e72d5..022f0ea85 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -39,15 +39,29 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     std::string doc() override
     {
         return R"(
+          # Description
+
           Copy the file or directory *path* to the Nix store, and
           print the resulting store path on standard output.
-        )";
-    }
 
-    Examples examples() override
-    {
-        return {
-        };
+          > **Warning**
+          >
+          > The resulting store path is not registered as a garbage
+          > collector root, so it could be deleted before you have a
+          > chance to register it.
+
+          # Examples
+
+          Add a regular file to the store:
+
+          ```console
+          # echo foo > bar
+          # nix add-to-store --flat ./bar
+          /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
+          # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
+          foo
+          ```
+        )";
     }
 
     Category category() override { return catUtility; }

From 1b0ca3866bae7185e628fda956634c52a5da7a15 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 14:23:18 +0100
Subject: [PATCH 528/882] nix add-to-store: Move markdown docs into a separate
 file

---
 src/nix/add-to-store.cc | 27 +++------------------------
 src/nix/add-to-store.md | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 24 deletions(-)
 create mode 100644 src/nix/add-to-store.md

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 022f0ea85..a2721431e 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -38,30 +38,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
     std::string doc() override
     {
-        return R"(
-          # Description
-
-          Copy the file or directory *path* to the Nix store, and
-          print the resulting store path on standard output.
-
-          > **Warning**
-          >
-          > The resulting store path is not registered as a garbage
-          > collector root, so it could be deleted before you have a
-          > chance to register it.
-
-          # Examples
-
-          Add a regular file to the store:
-
-          ```console
-          # echo foo > bar
-          # nix add-to-store --flat ./bar
-          /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
-          # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
-          foo
-          ```
-        )";
+        return
+          #include "add-to-store.md"
+          ;
     }
 
     Category category() override { return catUtility; }
diff --git a/src/nix/add-to-store.md b/src/nix/add-to-store.md
new file mode 100644
index 000000000..593ad67ad
--- /dev/null
+++ b/src/nix/add-to-store.md
@@ -0,0 +1,28 @@
+R""(
+
+# Description
+
+Copy the file or directory *path* to the Nix store, and
+print the resulting store path on standard output.
+
+> **Warning**
+>
+> The resulting store path is not registered as a garbage
+> collector root, so it could be deleted before you have a
+> chance to register it.
+
+# Examples
+
+Add a regular file to the store:
+
+```console
+# echo foo > bar
+
+# nix add-to-store --flat ./bar
+/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
+
+# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
+foo
+```
+
+)""

From 8ad2c9c4b97f291982598e34530122612c580b83 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 14:35:42 +0100
Subject: [PATCH 529/882] Remove 'dist' target

We're not producing source tarballs anymore so this has been
bitrotting.
---
 doc/manual/local.mk  |  2 --
 local.mk             |  6 ------
 mk/dist.mk           | 17 -----------------
 mk/lib.mk            |  4 ----
 mk/libraries.mk      |  1 -
 mk/programs.mk       |  1 -
 nix-rust/local.mk    |  2 --
 src/libexpr/local.mk |  2 --
 8 files changed, 35 deletions(-)
 delete mode 100644 mk/dist.mk

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index b40fa4ed2..ee5b328de 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -13,8 +13,6 @@ man-pages := $(foreach n, \
 
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
-dist-files += $(man-pages)
-
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
 dummy-env = env -i \
 	HOME=/dummy \
diff --git a/local.mk b/local.mk
index fe314b902..6a7074e8e 100644
--- a/local.mk
+++ b/local.mk
@@ -1,9 +1,3 @@
-ifeq ($(MAKECMDGOALS), dist)
-  dist-files += $(shell cat .dist-files)
-endif
-
-dist-files += configure config.h.in perl/configure
-
 clean-files += Makefile.config
 
 GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
diff --git a/mk/dist.mk b/mk/dist.mk
deleted file mode 100644
index 794b27771..000000000
--- a/mk/dist.mk
+++ /dev/null
@@ -1,17 +0,0 @@
-ifdef PACKAGE_NAME
-
-dist-name = $(PACKAGE_NAME)-$(PACKAGE_VERSION)
-
-dist: $(dist-name).tar.bz2 $(dist-name).tar.xz
-
-$(dist-name).tar.bz2: $(dist-files)
-	$(trace-gen) tar cfj $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,'
-
-$(dist-name).tar.xz: $(dist-files)
-	$(trace-gen) tar cfJ $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,'
-
-clean-files += $(dist-name).tar.bz2 $(dist-name).tar.xz
-
-print-top-help += echo "  dist: Generate a source distribution";
-
-endif
diff --git a/mk/lib.mk b/mk/lib.mk
index 1da51d879..a09ebaa97 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -10,7 +10,6 @@ bin-scripts :=
 noinst-scripts :=
 man-pages :=
 install-tests :=
-dist-files :=
 OS = $(shell uname -s)
 
 
@@ -112,9 +111,6 @@ $(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
 $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
 
 
-include mk/dist.mk
-
-
 .PHONY: default all man help
 
 all: $(programs-list) $(libs-list) $(jars-list) $(man-pages)
diff --git a/mk/libraries.mk b/mk/libraries.mk
index e6ef2e3ec..7c0e4f100 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -159,5 +159,4 @@ define build-library
   libs-list += $$($(1)_PATH)
   endif
   clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
-  dist-files += $$(_srcs)
 endef
diff --git a/mk/programs.mk b/mk/programs.mk
index 3fa9685c3..d0cf5baf0 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -79,7 +79,6 @@ define build-program
 
   programs-list += $$($(1)_PATH)
   clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
-  dist-files += $$(_srcs)
 
   # Phony target to run this program (typically as a dependency of 'check').
   .PHONY: $(1)_RUN
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index e4bfde31b..50db4783c 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -30,8 +30,6 @@ ifeq ($(OS), Darwin)
 	install_name_tool -id $@ $@
 endif
 
-dist-files += $(d)/vendor
-
 clean: clean-rust
 
 clean-rust:
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index a5422169d..519da33f7 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -35,8 +35,6 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
-
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \

From b2d6c6161e61c47f4d8a3faa5799160d100f4e8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 18:44:43 +0200
Subject: [PATCH 530/882] Move 'nix hash-*' and 'nix to-*' to 'nix hash'

From the 'nix' UX review.
---
 src/nix/hash.cc    | 52 +++++++++++++++++++++++++++++++++-------------
 tests/brotli.sh    |  4 ++--
 tests/fetchurl.sh  |  8 +++----
 tests/hash.sh      | 10 ++++-----
 tests/pure-eval.sh |  2 +-
 tests/tarball.sh   |  2 +-
 6 files changed, 51 insertions(+), 27 deletions(-)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 7f3d5717a..101b67e6a 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -8,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdHash : Command
+struct CmdHashBase : Command
 {
     FileIngestionMethod mode;
     Base base = SRI;
@@ -17,7 +17,7 @@ struct CmdHash : Command
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
-    CmdHash(FileIngestionMethod mode) : mode(mode)
+    CmdHashBase(FileIngestionMethod mode) : mode(mode)
     {
         mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
@@ -51,8 +51,6 @@ struct CmdHash : Command
         return d;
     }
 
-    Category category() override { return catUtility; }
-
     void run() override
     {
         for (auto path : paths) {
@@ -79,9 +77,6 @@ struct CmdHash : Command
     }
 };
 
-static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
-static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
-
 struct CmdToBase : Command
 {
     Base base;
@@ -103,8 +98,6 @@ struct CmdToBase : Command
             "SRI");
     }
 
-    Category category() override { return catUtility; }
-
     void run() override
     {
         for (auto s : args)
@@ -112,10 +105,41 @@ struct CmdToBase : Command
     }
 };
 
-static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref<CmdToBase>(Base16); });
-static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref<CmdToBase>(Base32); });
-static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref<CmdToBase>(Base64); });
-static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref<CmdToBase>(SRI); });
+struct CmdHash : NixMultiCommand
+{
+    CmdHash()
+        : MultiCommand({
+                {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
+                {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
+                {"to-base16", []() { return make_ref<CmdToBase>(Base16); }},
+                {"to-base32", []() { return make_ref<CmdToBase>(Base32); }},
+                {"to-base64", []() { return make_ref<CmdToBase>(Base64); }},
+                {"to-sri", []() { return make_ref<CmdToBase>(SRI); }},
+          })
+    { }
+
+    std::string description() override
+    {
+        return "compute and convert cryptographic hashes";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix hash' requires a sub-command.");
+        command->second->prepare();
+        command->second->run();
+    }
+
+    void printHelp(const string & programName, std::ostream & out) override
+    {
+        MultiCommand::printHelp(programName, out);
+    }
+};
+
+static auto rCmdHash = registerCommand<CmdHash>("hash");
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
@@ -149,7 +173,7 @@ static int compatNixHash(int argc, char * * argv)
     });
 
     if (op == opHash) {
-        CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
+        CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
         cmd.ht = ht;
         cmd.base = base32 ? Base32 : Base16;
         cmd.truncate = truncate;
diff --git a/tests/brotli.sh b/tests/brotli.sh
index a3c6e55a8..dc9bbdb66 100644
--- a/tests/brotli.sh
+++ b/tests/brotli.sh
@@ -9,13 +9,13 @@ outPath=$(nix-build dependencies.nix --no-out-link)
 
 nix copy --to $cacheURI $outPath
 
-HASH=$(nix hash-path $outPath)
+HASH=$(nix hash path $outPath)
 
 clearStore
 clearCacheCache
 
 nix copy --from $cacheURI $outPath --no-check-sigs
 
-HASH2=$(nix hash-path $outPath)
+HASH2=$(nix hash path $outPath)
 
 [[ $HASH = $HASH2 ]]
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 0f2044342..89ff08d4d 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -12,7 +12,7 @@ cmp $outPath fetchurl.sh
 # Now using a base-64 hash.
 clearStore
 
-hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
+hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh)
 
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
 
@@ -21,7 +21,7 @@ cmp $outPath fetchurl.sh
 # Now using an SRI hash.
 clearStore
 
-hash=$(nix hash-file ./fetchurl.sh)
+hash=$(nix hash file ./fetchurl.sh)
 
 [[ $hash =~ ^sha256- ]]
 
@@ -34,14 +34,14 @@ clearStore
 
 other_store=file://$TEST_ROOT/other_store?store=/fnord/store
 
-hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
+hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
 
 storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
 
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
-nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
+nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
           --no-out-link --substituters $other_store
 
 # Test unpacking a NAR.
diff --git a/tests/hash.sh b/tests/hash.sh
index 4cfc97901..e5f75e2cf 100644
--- a/tests/hash.sh
+++ b/tests/hash.sh
@@ -2,7 +2,7 @@ source common.sh
 
 try () {
     printf "%s" "$2" > $TEST_ROOT/vector
-    hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector)
+    hash=$(nix hash file --base16 $EXTRA --type "$1" $TEST_ROOT/vector)
     if test "$hash" != "$3"; then
         echo "hash $1, expected $3, got $hash"
         exit 1
@@ -69,17 +69,17 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 
 # Conversion.
 try3() {
-    h64=$(nix to-base64 --type "$1" "$2")
+    h64=$(nix hash to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
-    sri=$(nix to-sri --type "$1" "$2")
+    sri=$(nix hash to-sri --type "$1" "$2")
     [ "$sri" = "$1-$4" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
-    h16=$(nix to-base16 --type "$1" "$h64")
+    h16=$(nix hash to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
-    h16=$(nix to-base16 "$sri")
+    h16=$(nix hash to-base16 "$sri")
     [ "$h16" = "$2" ]
 }
 try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8="
diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh
index 561ca53fb..c994fbb98 100644
--- a/tests/pure-eval.sh
+++ b/tests/pure-eval.sh
@@ -15,7 +15,7 @@ nix eval --expr 'assert 1 + 2 == 3; true'
 
 [[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]]
 (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x")
-nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x"
+nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash file pure-eval.nix --type sha256)\"; })).x"
 
 rm -rf $TEST_ROOT/eval-out
 nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }'
diff --git a/tests/tarball.sh b/tests/tarball.sh
index fe65a22e4..d53ec8cd9 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -10,7 +10,7 @@ mkdir -p $tarroot
 cp dependencies.nix $tarroot/default.nix
 cp config.nix dependencies.builder*.sh $tarroot/
 
-hash=$(nix hash-path $tarroot)
+hash=$(nix hash path $tarroot)
 
 test_tarball() {
     local ext="$1"

From 5781f45c46d7f6c55f87528ab1e754d070bc99ee Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 17:55:27 +0100
Subject: [PATCH 531/882] Allow registering subcommands of subcommands

---
 src/nix/command.cc | 16 +++++++++++++++-
 src/nix/command.hh | 13 +++++++++++--
 src/nix/main.cc    |  2 +-
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 9a38c77f1..596217775 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -11,7 +11,21 @@ extern char * * environ __attribute__((weak));
 
 namespace nix {
 
-Commands * RegisterCommand::commands = nullptr;
+RegisterCommand::Commands * RegisterCommand::commands = nullptr;
+
+nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
+{
+    nix::Commands res;
+    for (auto & [name, command] : *RegisterCommand::commands)
+        if (name.size() == prefix.size() + 1) {
+            bool equal = true;
+            for (size_t i = 0; i < prefix.size(); ++i)
+                if (name[i] != prefix[i]) equal = false;
+            if (equal)
+                res.insert_or_assign(name[prefix.size()], command);
+        }
+    return res;
+}
 
 void NixMultiCommand::printHelp(const string & programName, std::ostream & out)
 {
diff --git a/src/nix/command.hh b/src/nix/command.hh
index d60c8aeb6..6882db195 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -176,20 +176,29 @@ struct StorePathCommand : public InstallablesCommand
 /* A helper class for registering commands globally. */
 struct RegisterCommand
 {
+    typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
     static Commands * commands;
 
-    RegisterCommand(const std::string & name,
+    RegisterCommand(std::vector<std::string> && name,
         std::function<ref<Command>()> command)
     {
         if (!commands) commands = new Commands;
         commands->emplace(name, command);
     }
+
+    static nix::Commands getCommandsFor(const std::vector<std::string> & prefix);
 };
 
 template<class T>
 static RegisterCommand registerCommand(const std::string & name)
 {
-    return RegisterCommand(name, [](){ return make_ref<T>(); });
+    return RegisterCommand({name}, [](){ return make_ref<T>(); });
+}
+
+template<class T>
+static RegisterCommand registerCommand2(std::vector<std::string> && name)
+{
+    return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
 Buildables build(ref<Store> store, Realise mode,
diff --git a/src/nix/main.cc b/src/nix/main.cc
index a75f8ae65..6d8c66406 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -59,7 +59,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     bool useNet = true;
     bool refresh = false;
 
-    NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
+    NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
     {
         categories.clear();
         categories[Command::catDefault] = "Main commands";

From 79c1967ded92574129c6a20116ef205a9c747bac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 18:06:46 +0100
Subject: [PATCH 532/882] Introduce 'nix store' command

---
 src/nix/store.cc | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 src/nix/store.cc

diff --git a/src/nix/store.cc b/src/nix/store.cc
new file mode 100644
index 000000000..e91bcc503
--- /dev/null
+++ b/src/nix/store.cc
@@ -0,0 +1,31 @@
+#include "command.hh"
+
+using namespace nix;
+
+struct CmdStore : virtual NixMultiCommand
+{
+    CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"}))
+    { }
+
+    std::string description() override
+    {
+        return "manipulate a Nix store";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix store' requires a sub-command.");
+        command->second->prepare();
+        command->second->run();
+    }
+
+    void printHelp(const string & programName, std::ostream & out) override
+    {
+        MultiCommand::printHelp(programName, out);
+    }
+};
+
+static auto rCmdStore = registerCommand<CmdStore>("store");

From ef583303f0720d8bc9d6351cd769f92d5dd678f3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 20:42:24 +0200
Subject: [PATCH 533/882] Move NAR-related commands to 'nix nar'

---
 src/nix/cat.cc         |  6 ++----
 src/nix/ls.cc          |  8 +++-----
 src/nix/nar.cc         | 31 +++++++++++++++++++++++++++++++
 tests/binary-cache.sh  |  6 +++---
 tests/linux-sandbox.sh |  4 ++--
 tests/nar-access.sh    | 26 +++++++++++++-------------
 6 files changed, 54 insertions(+), 27 deletions(-)
 create mode 100644 src/nix/nar.cc

diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index eef172cfc..4fa1c9491 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -64,13 +64,11 @@ struct CmdCatNar : StoreCommand, MixCat
         return "print the contents of a file inside a NAR file on stdout";
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store) override
     {
         cat(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
     }
 };
 
-static auto rCmdCatStore = registerCommand<CmdCatStore>("cat-store");
-static auto rCmdCatNar = registerCommand<CmdCatNar>("cat-nar");
+static auto rCmdCatStore = registerCommand2<CmdCatStore>({"store", "cat"});
+static auto rCmdCatNar = registerCommand2<CmdCatNar>({"nar", "cat"});
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index f39fdb2fd..d5fec4d84 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -134,7 +134,7 @@ struct CmdLsNar : Command, MixLs
         return {
             Example{
                 "To list a specific file in a NAR:",
-                "nix ls-nar -l hello.nar /bin/hello"
+                "nix nar ls -l hello.nar /bin/hello"
             },
         };
     }
@@ -144,13 +144,11 @@ struct CmdLsNar : Command, MixLs
         return "show information about a path inside a NAR file";
     }
 
-    Category category() override { return catUtility; }
-
     void run() override
     {
         list(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
     }
 };
 
-static auto rCmdLsStore = registerCommand<CmdLsStore>("ls-store");
-static auto rCmdLsNar = registerCommand<CmdLsNar>("ls-nar");
+static auto rCmdLsStore = registerCommand2<CmdLsStore>({"store", "ls"});
+static auto rCmdLsNar = registerCommand2<CmdLsNar>({"nar", "ls"});
diff --git a/src/nix/nar.cc b/src/nix/nar.cc
new file mode 100644
index 000000000..e239ce96a
--- /dev/null
+++ b/src/nix/nar.cc
@@ -0,0 +1,31 @@
+#include "command.hh"
+
+using namespace nix;
+
+struct CmdNar : NixMultiCommand
+{
+    CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"}))
+    { }
+
+    std::string description() override
+    {
+        return "query the contents of NAR files";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix nar' requires a sub-command.");
+        command->second->prepare();
+        command->second->run();
+    }
+
+    void printHelp(const string & programName, std::ostream & out) override
+    {
+        MultiCommand::printHelp(programName, out);
+    }
+};
+
+static auto rCmdNar = registerCommand<CmdNar>("nar");
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index e3b3982fe..8cb17caf8 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -196,13 +196,13 @@ narCache=$TEST_ROOT/nar-cache
 rm -rf $narCache
 mkdir $narCache
 
-[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
+[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
 
 rm -rfv "$cacheDir/nar"
 
-[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
+[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
 
-(! nix cat-store --store file://$cacheDir $outPath/foobar)
+(! nix store cat --store file://$cacheDir $outPath/foobar)
 
 
 # Test NAR listing generation.
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 16abd974c..70a90a907 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -22,9 +22,9 @@ outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store)
 
 nix path-info -r $outPath | grep input-2
 
-nix ls-store -R -l $outPath | grep foobar
+nix store ls -R -l $outPath | grep foobar
 
-nix cat-store $outPath/foobar | grep FOOBAR
+nix store cat $outPath/foobar | grep FOOBAR
 
 # Test --check without hash rewriting.
 nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
diff --git a/tests/nar-access.sh b/tests/nar-access.sh
index 88b997ca6..dcc2e8a36 100644
--- a/tests/nar-access.sh
+++ b/tests/nar-access.sh
@@ -9,45 +9,45 @@ cd "$TEST_ROOT"
 narFile="$TEST_ROOT/path.nar"
 nix-store --dump $storePath > $narFile
 
-# Check that find and ls-nar match.
+# Check that find and nar ls match.
 ( cd $storePath; find . | sort ) > files.find
-nix ls-nar -R -d $narFile "" | sort > files.ls-nar
+nix nar ls -R -d $narFile "" | sort > files.ls-nar
 diff -u files.find files.ls-nar
 
 # Check that file contents of data match.
-nix cat-nar $narFile /foo/data > data.cat-nar
+nix nar cat $narFile /foo/data > data.cat-nar
 diff -u data.cat-nar $storePath/foo/data
 
 # Check that file contents of baz match.
-nix cat-nar $narFile /foo/baz > baz.cat-nar
+nix nar cat $narFile /foo/baz > baz.cat-nar
 diff -u baz.cat-nar $storePath/foo/baz
 
-nix cat-store $storePath/foo/baz > baz.cat-nar
+nix store cat $storePath/foo/baz > baz.cat-nar
 diff -u baz.cat-nar $storePath/foo/baz
 
 # Test --json.
 diff -u \
-    <(nix ls-nar --json $narFile / | jq -S) \
+    <(nix nar ls --json $narFile / | jq -S) \
     <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
 diff -u \
-    <(nix ls-nar --json -R $narFile /foo | jq -S) \
+    <(nix nar ls --json -R $narFile /foo | jq -S) \
     <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S)
 diff -u \
-    <(nix ls-nar --json -R $narFile /foo/bar | jq -S) \
+    <(nix nar ls --json -R $narFile /foo/bar | jq -S) \
     <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S)
 diff -u \
-    <(nix ls-store --json $storePath | jq -S) \
+    <(nix store ls --json $storePath | jq -S) \
     <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
 diff -u \
-    <(nix ls-store --json -R $storePath/foo | jq -S) \
+    <(nix store ls --json -R $storePath/foo | jq -S) \
     <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S)
 diff -u \
-    <(nix ls-store --json -R $storePath/foo/bar| jq -S) \
+    <(nix store ls --json -R $storePath/foo/bar| jq -S) \
     <(echo '{"type":"regular","size":0}' | jq -S)
 
 # Test missing files.
-nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
-nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist'
+nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
+nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist'
 
 # Test failure to dump.
 if nix-store --dump $storePath >/dev/full ; then

From 0c15ae5d4b3366a14bca885e02599a941a334920 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 20:44:31 +0200
Subject: [PATCH 534/882] Add FIXME

---
 src/nix/show-derivation.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 6d4f295d7..8e1a58ac2 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -1,4 +1,5 @@
 // FIXME: integrate this with nix path-info?
+// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'?
 
 #include "command.hh"
 #include "common-args.hh"

From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 3 Dec 2020 15:35:38 -0600
Subject: [PATCH 535/882] Add extraPlatforms for Rosetta 2 macOS

macOS systems with ARM64 can utilize a translation layer at
/Library/Apple/usr/libexec/oah to run x86_64 binaries. This change
makes Nix recognize that and it to "extra-platforms". Note that there
are two cases here since Nix could be built for either x86_64 or
aarch64. In either case, we can switch to the other architecture.
Unfortunately there is not a good way to prevent aarch64 binaries from
being run in x86_64 contexts or vice versa - programs can always
execute programs for the other architecture.
---
 src/libstore/globals.cc | 22 ++++++++++++++++++++++
 src/libstore/globals.hh |  4 +++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index f38601d6d..59c49af8a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures()
     return features;
 }
 
+StringSet Settings::getDefaultExtraPlatforms()
+{
+    if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1())
+        return StringSet{"i686-linux"};
+#if __APPLE__
+    // Rosetta 2 emulation layer can run x86_64 binaries on aarch64
+    // machines. Note that we can’t force processes from executing
+    // x86_64 in aarch64 environments or vice versa since they can
+    // always exec with their own binary preferences.
+    else if (pathExists("/Library/Apple/usr/libexec/oah")) {
+        if (std::string{SYSTEM} == "x86_64-darwin")
+            return StringSet{"aarch64-darwin"};
+        else if (std::string{SYSTEM} == "aarch64-darwin")
+            return StringSet{"x86_64-darwin"};
+        else
+            return StringSet{};
+    }
+#endif
+    else
+        return StringSet{};
+}
+
 bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 {
     auto & f = experimentalFeatures.get();
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 4655ca058..8666a7d28 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -34,6 +34,8 @@ class Settings : public Config {
 
     StringSet getDefaultSystemFeatures();
 
+    StringSet getDefaultExtraPlatforms();
+
     bool isWSL1();
 
 public:
@@ -545,7 +547,7 @@ public:
 
     Setting<StringSet> extraPlatforms{
         this,
-        std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
+        getDefaultExtraPlatforms(),
         "extra-platforms",
         R"(
           Platforms other than the native one which this machine is capable of

From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 3 Dec 2020 15:41:59 -0600
Subject: [PATCH 536/882] Use posix_spawn_setbinpref_np to advise which
 architecture to run

When running universal binaries like /bin/bash, Darwin XNU will choose
which architecture of the binary to use based on "binary preferences".
This change sets that to the current platform for aarch64 and x86_64
builds. In addition it now uses posix_spawn instead of the usual
execve. Note, that this does not prevent the other architecture from
being run, just advises which to use.

Unfortunately, posix_spawnattr_setbinpref_np does not appear to be
inherited by child processes in x86_64 Rosetta 2 translations, meaning
that this will not always work as expected.

For example:

  {
    arm = derivation {
      name = "test";
      system = "aarch64-darwin";
      builder = "/bin/bash";
      args = [ "-e" (builtins.toFile "test" ''
        set -x
        /usr/sbin/sysctl sysctl.proc_translated
        /usr/sbin/sysctl sysctl.proc_native
        [ "$(/usr/bin/arch)" = arm64 ]
        /usr/bin/touch $out
      '') ];
    };
    rosetta = derivation {
      name = "test";
      system = "x86_64-darwin";
      builder = "/bin/bash";
      args = [ "-e" (builtins.toFile "test" ''
        set -x
        /usr/sbin/sysctl sysctl.proc_translated
        /usr/sbin/sysctl sysctl.proc_native
        [ "$(/usr/bin/arch)" = i386 ]
        echo It works!
        /usr/bin/touch $out
      '') ];
    };
  }

`arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on
aarch64-compiled Nix. I suspect there is a way to fix this since:

  $ /usr/bin/arch -arch x86_64 /bin/bash \
    -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch'
  arm64

seems to work correctly. We may need to wait for Apple to update
system_cmds in opensource.apple.com to find out how though.
---
 src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 1db85bd37..f370fd82d 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -50,6 +50,10 @@
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
+#if __APPLE__
+#include <spawn.h>
+#endif
+
 #include <pwd.h>
 #include <grp.h>
 
@@ -2844,7 +2848,27 @@ void DerivationGoal::runChild()
             }
         }
 
+#if __APPLE__
+        posix_spawnattr_t attrp;
+
+        if (posix_spawnattr_init(&attrp))
+            throw SysError("failed to initialize builder");
+
+        if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
+            throw SysError("failed to initialize builder");
+
+        if (drv->platform == "aarch64-darwin") {
+            cpu_type_t cpu = CPU_TYPE_ARM64;
+            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+        } else if (drv->platform == "x86_64-darwin") {
+            cpu_type_t cpu = CPU_TYPE_X86_64;
+            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+        }
+
+        posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#else
         execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#endif
 
         throw SysError("executing '%1%'", drv->builder);
 

From af373c2ece2c14ac652313a6f370dc344c85f86e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 22:45:44 +0100
Subject: [PATCH 537/882] Add deprecated aliases for renamed commands

---
 src/libutil/args.cc |  5 +----
 src/libutil/args.hh |  5 +++--
 src/nix/main.cc     | 30 +++++++++++++++++++++++++++++-
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 8bd9c8aeb..61f9503ec 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -86,6 +86,7 @@ void Args::parseCmdline(const Strings & _cmdline)
                 throw UsageError("unrecognised flag '%1%'", arg);
         }
         else {
+            pos = rewriteArgs(cmdline, pos);
             pendingArgs.push_back(*pos++);
             if (processArgs(pendingArgs, false))
                 pendingArgs.clear();
@@ -390,10 +391,6 @@ MultiCommand::MultiCommand(const Commands & commands)
         .optional = true,
         .handler = {[=](std::string s) {
             assert(!command);
-            if (auto alias = get(deprecatedAliases, s)) {
-                warn("'%s' is a deprecated alias for '%s'", s, *alias);
-                s = *alias;
-            }
             if (auto prefix = needsCompletion(s)) {
                 for (auto & [name, command] : commands)
                     if (hasPrefix(name, *prefix))
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 26f1bc11b..8069fd70f 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -115,6 +115,9 @@ protected:
 
     virtual bool processArgs(const Strings & args, bool finish);
 
+    virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos)
+    { return pos; }
+
     std::set<std::string> hiddenCategories;
 
 public:
@@ -257,8 +260,6 @@ public:
 
     std::map<Command::Category, std::string> categories;
 
-    std::map<std::string, std::string> deprecatedAliases;
-
     // Selected command, if any.
     std::optional<std::pair<std::string, ref<Command>>> command;
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 6d8c66406..94f4cad3c 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -112,8 +112,36 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .description = "consider all previously downloaded files out-of-date",
             .handler = {[&]() { refresh = true; }},
         });
+    }
 
-        deprecatedAliases.insert({"dev-shell", "develop"});
+    std::map<std::string, std::vector<std::string>> aliases = {
+        {"dev-shell", {"develop"}},
+        {"hash-file", {"hash", "file"}},
+        {"hash-path", {"hash", "path"}},
+        {"to-base16", {"hash", "to-base16"}},
+        {"to-base32", {"hash", "to-base32"}},
+        {"to-base64", {"hash", "to-base64"}},
+        {"ls-nar", {"nar", "ls"}},
+        {"ls-store", {"store", "ls"}},
+        {"cat-nar", {"nar", "cat"}},
+        {"cat-store", {"store", "cat"}},
+    };
+
+    bool aliasUsed = false;
+
+    Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override
+    {
+        if (aliasUsed || command || pos == args.end()) return pos;
+        auto arg = *pos;
+        auto i = aliases.find(arg);
+        if (i == aliases.end()) return pos;
+        warn("'%s' is a deprecated alias for '%s'",
+            arg, concatStringsSep(" ", i->second));
+        pos = args.erase(pos);
+        for (auto j = i->second.rbegin(); j != i->second.rend(); ++j)
+            pos = args.insert(pos, *j);
+        aliasUsed = true;
+        return pos;
     }
 
     void printFlags(std::ostream & out) override

From a1cd805cba7a4408e75779bc4099f92e81fd6ac7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 21:04:26 +0200
Subject: [PATCH 538/882] Add 'nix nar dump-path'

This only differs from 'nix store dump-path' in that the path doesn't
need to be a store path.
---
 src/nix/dump-path.cc | 44 ++++++++++++++++++++++++++++++++++++++++++--
 src/nix/main.cc      |  1 +
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 6fd197531..4b225ae9f 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -1,5 +1,6 @@
 #include "command.hh"
 #include "store-api.hh"
+#include "archive.hh"
 
 using namespace nix;
 
@@ -7,7 +8,7 @@ struct CmdDumpPath : StorePathCommand
 {
     std::string description() override
     {
-        return "dump a store path to stdout (in NAR format)";
+        return "serialise a store path to stdout in NAR format";
     }
 
     Examples examples() override
@@ -30,4 +31,43 @@ struct CmdDumpPath : StorePathCommand
     }
 };
 
-static auto rDumpPath = registerCommand<CmdDumpPath>("dump-path");
+
+static auto rDumpPath = registerCommand2<CmdDumpPath>({"store", "dump-path"});
+
+struct CmdDumpPath2 : Command
+{
+    Path path;
+
+    CmdDumpPath2()
+    {
+        expectArgs({
+            .label = "path",
+            .handler = {&path},
+            .completer = completePath
+        });
+    }
+
+    std::string description() override
+    {
+        return "serialise a path to stdout in NAR format";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To serialise directory 'foo' as a NAR:",
+                "nix nar dump-path ./foo"
+            },
+        };
+    }
+
+    void run() override
+    {
+        FdSink sink(STDOUT_FILENO);
+        dumpPath(path, sink);
+        sink.flush();
+    }
+};
+
+static auto rDumpPath2 = registerCommand2<CmdDumpPath2>({"nar", "dump-path"});
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 94f4cad3c..0002be291 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -125,6 +125,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         {"ls-store", {"store", "ls"}},
         {"cat-nar", {"nar", "cat"}},
         {"cat-store", {"store", "cat"}},
+        {"dump-path", {"store", "dump-path"}},
     };
 
     bool aliasUsed = false;

From ea2062a2d9144d78588675950fc04756f0d200a5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Jul 2020 20:38:56 +0200
Subject: [PATCH 539/882] Move most store-related commands to 'nix store'

---
 src/nix/add-to-store.cc             |  4 +--
 src/nix/cat.cc                      |  2 --
 src/nix/diff-closures.cc            |  6 ++--
 src/nix/dump-path.cc                |  5 +--
 src/nix/ls.cc                       |  4 +--
 src/nix/main.cc                     | 18 ++++++++---
 src/nix/make-content-addressable.cc |  8 ++---
 src/nix/optimise-store.cc           |  6 ++--
 src/nix/ping-store.cc               |  6 ++--
 src/nix/sigs.cc                     |  8 ++---
 src/nix/verify.cc                   |  8 ++---
 tests/binary-cache.sh               |  2 +-
 tests/fetchurl.sh                   |  2 +-
 tests/gc-auto.sh                    |  6 ++--
 tests/recursive.sh                  |  4 +--
 tests/signing.sh                    | 48 ++++++++++++++---------------
 tests/ssh-relay.sh                  |  2 +-
 17 files changed, 62 insertions(+), 77 deletions(-)

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index a2721431e..66822b0ff 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -43,8 +43,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand
           ;
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store) override
     {
         if (!namePart) namePart = baseNameOf(path);
@@ -80,4 +78,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     }
 };
 
-static auto rCmdAddToStore = registerCommand<CmdAddToStore>("add-to-store");
+static auto rCmdAddToStore = registerCommand2<CmdAddToStore>({"store", "add-path"});
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 4fa1c9491..2ecffc9a5 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -37,8 +37,6 @@ struct CmdCatStore : StoreCommand, MixCat
         return "print the contents of a file in the Nix store on stdout";
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store) override
     {
         cat(store->getFSAccessor());
diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc
index 30e7b20e1..f72b5eff7 100644
--- a/src/nix/diff-closures.cc
+++ b/src/nix/diff-closures.cc
@@ -121,14 +121,12 @@ struct CmdDiffClosures : SourceExprCommand
         return "show what packages and versions were added and removed between two closures";
     }
 
-    Category category() override { return catSecondary; }
-
     Examples examples() override
     {
         return {
             {
                 "To show what got added and removed between two versions of the NixOS system profile:",
-                "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
+                "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
             },
         };
     }
@@ -143,4 +141,4 @@ struct CmdDiffClosures : SourceExprCommand
     }
 };
 
-static auto rCmdDiffClosures = registerCommand<CmdDiffClosures>("diff-closures");
+static auto rCmdDiffClosures = registerCommand2<CmdDiffClosures>({"store", "diff-closures"});
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 4b225ae9f..256db64a9 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -16,13 +16,11 @@ struct CmdDumpPath : StorePathCommand
         return {
             Example{
                 "To get a NAR from the binary cache https://cache.nixos.org/:",
-                "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"
+                "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"
             },
         };
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store, const StorePath & storePath) override
     {
         FdSink sink(STDOUT_FILENO);
@@ -31,7 +29,6 @@ struct CmdDumpPath : StorePathCommand
     }
 };
 
-
 static auto rDumpPath = registerCommand2<CmdDumpPath>({"store", "dump-path"});
 
 struct CmdDumpPath2 : Command
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index d5fec4d84..1f5ed6913 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -97,7 +97,7 @@ struct CmdLsStore : StoreCommand, MixLs
         return {
             Example{
                 "To list the contents of a store path in a binary cache:",
-                "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"
+                "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"
             },
         };
     }
@@ -107,8 +107,6 @@ struct CmdLsStore : StoreCommand, MixLs
         return "show information about a path in the Nix store";
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store) override
     {
         list(store->getFSAccessor());
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 0002be291..fb3bffeaf 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -115,17 +115,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     }
 
     std::map<std::string, std::vector<std::string>> aliases = {
+        {"add-to-store", {"store", "add-path"}},
+        {"cat-nar", {"nar", "cat"}},
+        {"cat-store", {"store", "cat"}},
+        {"copy-sigs", {"store", "copy-sigs"}},
         {"dev-shell", {"develop"}},
+        {"diff-closures", {"store", "diff-closures"}},
+        {"dump-path", {"store", "dump-path"}},
         {"hash-file", {"hash", "file"}},
         {"hash-path", {"hash", "path"}},
+        {"ls-nar", {"nar", "ls"}},
+        {"ls-store", {"store", "ls"}},
+        {"make-content-addressable", {"store", "make-content-addressable"}},
+        {"optimise-store", {"store", "optimise"}},
+        {"ping-store", {"store", "ping"}},
+        {"sign-paths", {"store", "sign-paths"}},
         {"to-base16", {"hash", "to-base16"}},
         {"to-base32", {"hash", "to-base32"}},
         {"to-base64", {"hash", "to-base64"}},
-        {"ls-nar", {"nar", "ls"}},
-        {"ls-store", {"store", "ls"}},
-        {"cat-nar", {"nar", "cat"}},
-        {"cat-store", {"store", "cat"}},
-        {"dump-path", {"store", "dump-path"}},
+        {"verify", {"store", "verify"}},
     };
 
     bool aliasUsed = false;
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 12c2cf776..0dade90ef 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -23,17 +23,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
         return {
             Example{
                 "To create a content-addressable representation of GNU Hello (but not its dependencies):",
-                "nix make-content-addressable nixpkgs#hello"
+                "nix store make-content-addressable nixpkgs#hello"
             },
             Example{
                 "To compute a content-addressable representation of the current NixOS system closure:",
-                "nix make-content-addressable -r /run/current-system"
+                "nix store make-content-addressable -r /run/current-system"
             },
         };
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store, StorePaths storePaths) override
     {
         auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
@@ -108,4 +106,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
     }
 };
 
-static auto rCmdMakeContentAddressable = registerCommand<CmdMakeContentAddressable>("make-content-addressable");
+static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index 51a7a9756..bc7f175ac 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -18,17 +18,15 @@ struct CmdOptimiseStore : StoreCommand
         return {
             Example{
                 "To optimise the Nix store:",
-                "nix optimise-store"
+                "nix store optimise"
             },
         };
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store) override
     {
         store->optimiseStore();
     }
 };
 
-static auto rCmdOptimiseStore = registerCommand<CmdOptimiseStore>("optimise-store");
+static auto rCmdOptimiseStore = registerCommand2<CmdOptimiseStore>({"store", "optimise"});
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 8db78d591..19b1a55c8 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -16,17 +16,15 @@ struct CmdPingStore : StoreCommand
         return {
             Example{
                 "To test whether connecting to a remote Nix store via SSH works:",
-                "nix ping-store --store ssh://mac1"
+                "nix store ping --store ssh://mac1"
             },
         };
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store) override
     {
         store->connect();
     }
 };
 
-static auto rCmdPingStore = registerCommand<CmdPingStore>("ping-store");
+static auto rCmdPingStore = registerCommand2<CmdPingStore>({"store", "ping"});
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 44916c77f..37b8a6712 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -27,8 +27,6 @@ struct CmdCopySigs : StorePathsCommand
         return "copy path signatures from substituters (like binary caches)";
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store, StorePaths storePaths) override
     {
         if (substituterUris.empty())
@@ -92,7 +90,7 @@ struct CmdCopySigs : StorePathsCommand
     }
 };
 
-static auto rCmdCopySigs = registerCommand<CmdCopySigs>("copy-sigs");
+static auto rCmdCopySigs = registerCommand2<CmdCopySigs>({"store", "copy-sigs"});
 
 struct CmdSignPaths : StorePathsCommand
 {
@@ -115,8 +113,6 @@ struct CmdSignPaths : StorePathsCommand
         return "sign the specified paths";
     }
 
-    Category category() override { return catUtility; }
-
     void run(ref<Store> store, StorePaths storePaths) override
     {
         if (secretKeyFile.empty())
@@ -144,4 +140,4 @@ struct CmdSignPaths : StorePathsCommand
     }
 };
 
-static auto rCmdSignPaths = registerCommand<CmdSignPaths>("sign-paths");
+static auto rCmdSignPaths = registerCommand2<CmdSignPaths>({"store", "sign-paths"});
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index ec7333d03..bcf85d7dd 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -40,17 +40,15 @@ struct CmdVerify : StorePathsCommand
         return {
             Example{
                 "To verify the entire Nix store:",
-                "nix verify --all"
+                "nix store verify --all"
             },
             Example{
                 "To check whether each path in the closure of Firefox has at least 2 signatures:",
-                "nix verify -r -n2 --no-contents $(type -p firefox)"
+                "nix store verify -r -n2 --no-contents $(type -p firefox)"
             },
         };
     }
 
-    Category category() override { return catSecondary; }
-
     void run(ref<Store> store, StorePaths storePaths) override
     {
         std::vector<ref<Store>> substituters;
@@ -189,4 +187,4 @@ struct CmdVerify : StorePathsCommand
     }
 };
 
-static auto rCmdVerify = registerCommand<CmdVerify>("verify");
+static auto rCmdVerify = registerCommand2<CmdVerify>({"store", "verify"});
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 8cb17caf8..92ed36225 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -188,7 +188,7 @@ unset _NIX_FORCE_HTTP
 
 
 # Test 'nix verify --all' on a binary cache.
-nix verify -vvvvv --all --store file://$cacheDir --no-trust
+nix store verify -vvvvv --all --store file://$cacheDir --no-trust
 
 
 # Test local NAR caching.
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 89ff08d4d..7ec25808c 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -36,7 +36,7 @@ other_store=file://$TEST_ROOT/other_store?store=/fnord/store
 
 hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
 
-storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
+storePath=$(nix --store $other_store store add-path --flat ./fetchurl.sh)
 
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 3add896c6..6867f2eb4 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -2,9 +2,9 @@ source common.sh
 
 clearStore
 
-garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh)
-garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh)
-garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh)
+garbage1=$(nix store add-path --name garbage1 ./nar-access.sh)
+garbage2=$(nix store add-path --name garbage2 ./nar-access.sh)
+garbage3=$(nix store add-path --name garbage3 ./nar-access.sh)
 
 ls -l $garbage3
 POSIXLY_CORRECT=1 du $garbage3
diff --git a/tests/recursive.sh b/tests/recursive.sh
index 80a178cc7..b020ec710 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -7,7 +7,7 @@ clearStore
 
 rm -f $TEST_ROOT/result
 
-export unreachable=$(nix add-to-store ./recursive.sh)
+export unreachable=$(nix store add-path ./recursive.sh)
 
 NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
   with import ./config.nix;
@@ -38,7 +38,7 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
 
       # Add something to the store.
       echo foobar > foobar
-      foobar=$(nix $opts add-to-store ./foobar)
+      foobar=$(nix $opts store add-path ./foobar)
 
       nix $opts path-info $foobar
       nix $opts build $foobar
diff --git a/tests/signing.sh b/tests/signing.sh
index 9e29e3fbf..bd6280cc6 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -17,40 +17,40 @@ info=$(nix path-info --json $outPath)
 [[ $info =~ 'cache1.example.org' ]]
 [[ $info =~ 'cache2.example.org' ]]
 
-# Test "nix verify".
-nix verify -r $outPath
+# Test "nix store verify".
+nix store verify -r $outPath
 
-expect 2 nix verify -r $outPath --sigs-needed 1
+expect 2 nix store verify -r $outPath --sigs-needed 1
 
-nix verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1
+nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1
 
-expect 2 nix verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1
+expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1
 
-nix verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
+nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
-nix verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
+nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
 # Build something unsigned.
 outPath2=$(nix-build simple.nix --no-out-link)
 
-nix verify -r $outPath
+nix store verify -r $outPath
 
 # Verify that the path did not get signed but does have the ultimate bit.
 info=$(nix path-info --json $outPath2)
 [[ $info =~ '"ultimate":true' ]]
 (! [[ $info =~ 'signatures' ]])
 
-# Test "nix verify".
-nix verify -r $outPath2
+# Test "nix store verify".
+nix store verify -r $outPath2
 
-expect 2 nix verify -r $outPath2 --sigs-needed 1
+expect 2 nix store verify -r $outPath2 --sigs-needed 1
 
-expect 2 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
+expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
-# Test "nix sign-paths".
-nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2
+# Test "nix store sign-paths".
+nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2
 
-nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
+nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
 # Build something content-addressed.
 outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link)
@@ -59,12 +59,12 @@ outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no
 
 # Content-addressed paths don't need signatures, so they verify
 # regardless of --sigs-needed.
-nix verify $outPathCA
-nix verify $outPathCA --sigs-needed 1000
+nix store verify $outPathCA
+nix store verify $outPathCA --sigs-needed 1000
 
 # Check that signing a content-addressed path doesn't overflow validSigs
-nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA
-nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1
+nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA
+nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1
 
 # Copy to a binary cache.
 nix copy --to file://$cacheDir $outPath2
@@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2)
 (! [[ $info =~ 'cache2.example.org' ]])
 
 # Verify that adding a signature to a path in a binary cache works.
-nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
+nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
 info=$(nix path-info --store file://$cacheDir --json $outPath2)
 [[ $info =~ 'cache1.example.org' ]]
 [[ $info =~ 'cache2.example.org' ]]
@@ -89,17 +89,17 @@ rm -rf $TEST_ROOT/store0
 # But succeed if we supply the public keys.
 nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1
 
-expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath
+expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath
 
-nix verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1
-nix verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
+nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1
+nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
 # It should also succeed if we disable signature checking.
 (! nix copy --to $TEST_ROOT/store0 $outPath2)
 nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2
 
 # But signatures should still get copied.
-nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1
+nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1
 
 # Content-addressed stuff can be copied without signatures.
 nix copy --to $TEST_ROOT/store0 $outPathCA
diff --git a/tests/ssh-relay.sh b/tests/ssh-relay.sh
index dce50974b..053b2f00d 100644
--- a/tests/ssh-relay.sh
+++ b/tests/ssh-relay.sh
@@ -11,6 +11,6 @@ store+=$remote_store
 store+=$remote_store
 store+=$remote_store
 
-out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh)
+out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh)
 
 [ foo = $(< $out) ]

From fa8dad10edcebd942ce99e6413fa38e3fe883f15 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 Dec 2020 23:26:23 +0100
Subject: [PATCH 540/882] Typo

---
 src/nix/profile.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 75426b2e3..8cf5ccd62 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -413,7 +413,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
         return {
             Example{
                 "To show what changed between each generation of the NixOS system profile:",
-                "nix profile diff-closure --profile /nix/var/nix/profiles/system"
+                "nix profile diff-closures --profile /nix/var/nix/profiles/system"
             },
         };
     }

From f337aa70998141ccfaa956e9f670152dbb15b385 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 4 Dec 2020 00:58:09 +0100
Subject: [PATCH 541/882] Split 'nix store add-to-store' into 'add-path' and
 'add-file'

This makes it consistent with 'nix hash <path|file>'.
---
 src/nix/{add-to-store.md => add-file.md} |  6 +--
 src/nix/add-path.md                      | 29 +++++++++++
 src/nix/add-to-store.cc                  | 65 ++++++++++++++++--------
 tests/fetchurl.sh                        |  2 +-
 4 files changed, 77 insertions(+), 25 deletions(-)
 rename src/nix/{add-to-store.md => add-file.md} (72%)
 create mode 100644 src/nix/add-path.md

diff --git a/src/nix/add-to-store.md b/src/nix/add-file.md
similarity index 72%
rename from src/nix/add-to-store.md
rename to src/nix/add-file.md
index 593ad67ad..ed237a035 100644
--- a/src/nix/add-to-store.md
+++ b/src/nix/add-file.md
@@ -2,8 +2,8 @@ R""(
 
 # Description
 
-Copy the file or directory *path* to the Nix store, and
-print the resulting store path on standard output.
+Copy the regular file *path* to the Nix store, and print the resulting
+store path on standard output.
 
 > **Warning**
 >
@@ -18,7 +18,7 @@ Add a regular file to the store:
 ```console
 # echo foo > bar
 
-# nix add-to-store --flat ./bar
+# nix store add-file ./bar
 /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
 
 # cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
diff --git a/src/nix/add-path.md b/src/nix/add-path.md
new file mode 100644
index 000000000..87473611d
--- /dev/null
+++ b/src/nix/add-path.md
@@ -0,0 +1,29 @@
+R""(
+
+# Description
+
+Copy *path* to the Nix store, and print the resulting store path on
+standard output.
+
+> **Warning**
+>
+> The resulting store path is not registered as a garbage
+> collector root, so it could be deleted before you have a
+> chance to register it.
+
+# Examples
+
+Add a directory to the store:
+
+```console
+# mkdir dir
+# echo foo > dir/bar
+
+# nix store add-path ./dir
+/nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir
+
+# cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar
+foo
+```
+
+)""
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 66822b0ff..ea4bbbab9 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,10 +9,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
     std::optional<std::string> namePart;
-    FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
+    FileIngestionMethod ingestionMethod;
 
     CmdAddToStore()
     {
+        // FIXME: completion
         expectArg("path", &path);
 
         addFlag({
@@ -22,25 +23,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             .labels = {"name"},
             .handler = {&namePart},
         });
-
-        addFlag({
-            .longName = "flat",
-            .shortName = 0,
-            .description = "add flat file to the Nix store",
-            .handler = {&ingestionMethod, FileIngestionMethod::Flat},
-        });
-    }
-
-    std::string description() override
-    {
-        return "add a path to the Nix store";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "add-to-store.md"
-          ;
     }
 
     void run(ref<Store> store) override
@@ -78,4 +60,45 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     }
 };
 
-static auto rCmdAddToStore = registerCommand2<CmdAddToStore>({"store", "add-path"});
+struct CmdAddFile : CmdAddToStore
+{
+    CmdAddFile()
+    {
+        ingestionMethod = FileIngestionMethod::Flat;
+    }
+
+    std::string description() override
+    {
+        return "add a regular file to the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "add-file.md"
+          ;
+    }
+};
+
+struct CmdAddPath : CmdAddToStore
+{
+    CmdAddPath()
+    {
+        ingestionMethod = FileIngestionMethod::Recursive;
+    }
+
+    std::string description() override
+    {
+        return "add a path to the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "add-path.md"
+          ;
+    }
+};
+
+static auto rCmdAddFile = registerCommand2<CmdAddFile>({"store", "add-file"});
+static auto rCmdAddPath = registerCommand2<CmdAddPath>({"store", "add-path"});
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 7ec25808c..10ec0173a 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -36,7 +36,7 @@ other_store=file://$TEST_ROOT/other_store?store=/fnord/store
 
 hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
 
-storePath=$(nix --store $other_store store add-path --flat ./fetchurl.sh)
+storePath=$(nix --store $other_store store add-file ./fetchurl.sh)
 
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 

From be09af80026b751569e7dd6519d1564e10e4f34b Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 3 Dec 2020 18:03:30 -0600
Subject: [PATCH 542/882] Include static "nix" binary in Hydra build products

This allows users to get Nix from Hydra via a stable url like
https://hydra.nixos.org/build/132078238/download/1/nix
---
 flake.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flake.nix b/flake.nix
index 7311a2471..9addccd63 100644
--- a/flake.nix
+++ b/flake.nix
@@ -466,6 +466,8 @@
           postInstall = ''
             mkdir -p $doc/nix-support
             echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+            mkdir -p $out/nix-support
+            echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
           '';
 
           doInstallCheck = true;

From 5f66edf24503fabf410bb923de761131cea57771 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 4 Dec 2020 14:28:27 +0100
Subject: [PATCH 543/882] Make `make install` less noisy

Remove the printing and useless output of a couple of commands when running `make install`
---
 doc/manual/local.mk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index ee5b328de..81a7755e8 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -68,14 +68,14 @@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix
 
 $(d)/builtins.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
-	mv $@.tmp $@
+	@mv $@.tmp $@
 
 # Generate the HTML manual.
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
 install: $(d)/src/command-ref/new-cli
-	for i in doc/manual/src/command-ref/new-cli/*.md; do \
+	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
 	  name=$$(basename $$i .md); \
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
 	  printf "Title: %s\n\n" "$$name" > $$i.tmp; \
@@ -84,7 +84,7 @@ install: $(d)/src/command-ref/new-cli
 	done
 
 $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
-	$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
+	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual
 	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
 
 endif

From e20a3ec756c61b076d5fbaea6d976c8ee7fa4c07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phan=20Kochen?= <git@stephank.nl>
Date: Fri, 4 Dec 2020 19:32:35 +0100
Subject: [PATCH 544/882] Fix compatibility with newer AWS SDKs

Tested against AWS SDK 1.8.99. Fixes #3201.
---
 configure.ac                          | 1 +
 src/libstore/s3-binary-cache-store.cc | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/configure.ac b/configure.ac
index 39306b953..daf378997 100644
--- a/configure.ac
+++ b/configure.ac
@@ -255,6 +255,7 @@ if test -n "$enable_s3"; then
   declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
   AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
   AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
+  AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.])
 fi
 
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 4519dd5b5..27253fc12 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -57,6 +57,10 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
     {
         debug("AWS: %s", chomp(statement));
     }
+
+#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115)
+    void Flush() override {}
+#endif
 };
 
 static void initAWS()

From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 4 Dec 2020 13:26:53 -0600
Subject: [PATCH 545/882] Use com.apple.oahd.plist for rosetta 2 detection

---
 src/libstore/globals.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 59c49af8a..ad66ef8a8 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms()
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    else if (pathExists("/Library/Apple/usr/libexec/oah")) {
+    else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
         if (std::string{SYSTEM} == "x86_64-darwin")
             return StringSet{"aarch64-darwin"};
         else if (std::string{SYSTEM} == "aarch64-darwin")

From 692549c542f673015105a3aa4358c1e3095bb0e0 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 4 Dec 2020 13:28:09 -0600
Subject: [PATCH 546/882] Use com.apple.oahd.plist for rosetta 2 detection

---
 scripts/install.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install.in b/scripts/install.in
index 9c8831d2b..0eaf25bb3 100755
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -47,7 +47,7 @@ case "$(uname -s).$(uname -m)" in
         ;;
     Darwin.arm64|Darwin.aarch64)
         # check for Rosetta 2 support
-        if ! [ -d /Library/Apple/usr/libexec/oah ]; then
+        if ! [ -f /Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist ]; then
           oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation"
         fi
 

From b9a00fd15bb158c293b40aec49c9a426cc4c8921 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 4 Dec 2020 22:17:19 -0600
Subject: [PATCH 547/882] =?UTF-8?q?Canonicalize=20binary=20caches=20with?=
 =?UTF-8?q?=20=E2=80=98/=E2=80=99=20when=20one=20is=20missing?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This checks if there is a trusted substituter with a slash, so
trusting https://cache.nixos.org also implies https://cache.nixos.org/
is trusted.
---
 src/libstore/daemon.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index e5cfe94cb..2224d54d5 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -215,6 +215,8 @@ struct ClientSettings
                 for (auto & s : ss)
                     if (trusted.count(s))
                         subs.push_back(s);
+                    else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
+                        subs.push_back(s + "/");
                     else
                         warn("ignoring untrusted substituter '%s'", s);
                 res = subs;

From aa07502009625fe0d38fde1a23c50dd34f1996eb Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sun, 6 Dec 2020 23:04:42 -0600
Subject: [PATCH 548/882] Always default to cache.nixos.org even when different
 nix store dir

Since 0744f7f, it is now useful to have cache.nixos.org in substituers
even if /nix/store is not the Nix Store Dir. This can always be
overridden via configuration, though.
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 4655ca058..6b4775683 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -583,7 +583,7 @@ public:
 
     Setting<Strings> substituters{
         this,
-        nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
+        Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
           A list of URLs of substituters, separated by whitespace. The default

From 0d7714b0d7f626afc174f1faaf52725290a1247c Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Tue, 8 Dec 2020 10:25:03 +0100
Subject: [PATCH 549/882] forgot to add the files

---
 doc/manual/src/command-ref/nix.md           | 1 +
 doc/manual/src/contributing/contributing.md | 0
 2 files changed, 1 insertion(+)
 create mode 100644 doc/manual/src/command-ref/nix.md
 create mode 100644 doc/manual/src/contributing/contributing.md

diff --git a/doc/manual/src/command-ref/nix.md b/doc/manual/src/command-ref/nix.md
new file mode 100644
index 000000000..cc677d127
--- /dev/null
+++ b/doc/manual/src/command-ref/nix.md
@@ -0,0 +1 @@
+# nix
diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md
new file mode 100644
index 000000000..e69de29bb

From 1b1e0760335832c87516b9103b670b34662d5daf Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 8 Dec 2020 11:11:02 +0100
Subject: [PATCH 550/882] Re-query for the derivation outputs in the
 post-build-hook

We can't assume that the runtime state knows about them as they might have
been built remotely, in which case we must query the db again to get
them.
---
 src/libstore/build/derivation-goal.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 1db85bd37..fdf777c27 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -899,8 +899,10 @@ void DerivationGoal::buildDone()
                 Logger::Fields{worker.store.printStorePath(drvPath)});
             PushActivity pact(act.id);
             StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
+            for (auto& [_, maybeOutPath] :
+                 worker.store.queryPartialDerivationOutputMap(drvPath)) {
+                if (maybeOutPath)
+                    outputPaths.insert(*maybeOutPath);
             }
             std::map<std::string, std::string> hookEnvironment = getEnv();
 

From ae77f21474594fc4555bafa3291424b12ff3d4ac Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Tue, 8 Dec 2020 11:59:23 +0100
Subject: [PATCH 551/882] Switch away from classification as Tier1-3

to classification to a more descriptive classification.
---
 doc/manual/src/contributing/cli-guideline.md | 44 ++++++++++++++------
 doc/manual/src/contributing/contributing.md  |  1 +
 doc/manual/src/contributing/hacking.md       |  1 +
 3 files changed, 33 insertions(+), 13 deletions(-)
 create mode 100644 doc/manual/src/contributing/hacking.md

diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md
index 01df07136..0132867c8 100644
--- a/doc/manual/src/contributing/cli-guideline.md
+++ b/doc/manual/src/contributing/cli-guideline.md
@@ -58,12 +58,13 @@ them.
 
 This classification tries to separate commands in 3 categories in terms of
 their importance in regards to the new users. Users who are likely to be
-impacted the most by bad user experience. This does not mean that we will not
-take care of Tier3 commands, it means we will only put more helpful details for
-beginners into Tier1 commands.
+impacted the most by bad user experience.
+
+- **Main commands**
+
+  Commands used for our main use cases and most likely used by new users. We
+  expect attention to details, such as:
 
-- **Tier1**: Commands used for our main use cases and most likely used by new
-  users. From Tier1 commands we expect attention to details, such as:
     - Proper use of [colors](#colors), [emojis](#special-unicode-characters)
       and [aligning of text](#text-alignment).
     - [Autocomplete](#shell-completion) of options.
@@ -73,16 +74,32 @@ beginners into Tier1 commands.
       Nix ecosystem.
     - [Help pages](#help-is-essential) to be as good as we can write them
       pointing to external documentation and tutorials for more.
-- **Tier2**: Commands that are somewhere between Tier1 and Tier2, not really
-  exposing some implementation detail, but not something that we expect a user.
-  From Tier2 command we expect less attention to details, but still some:
+
+  Examples of such commands: `nix init`, `nix develop`, `nix build`, `nix run`,
+  ...
+
+- **Infrequently used commands**
+
+  From infrequently used commands we expect less attention to details, but
+  still some:
+
     - Proper use of [colors](#colors), [emojis](#special-unicode-characters)
       and [aligning of text](#text-alignment).
     - [Autocomplete](#shell-completion) of options.
-- **Tier3**: Commands that expose certain internal functionality of `nix`,
-  mostly used by other scripts.
+
+  Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ...
+
+- **Utility and scripting commands**
+
+  Commands that expose certain internal functionality of `nix`, mostly used by
+  other scripts.
+
     - [Autocomplete](#shell-completion) of options.
 
+  Examples of such commands: `nix store copy`, `nix hash base16`, `nix store
+  ping`, ...
+
+
 # Help is essential
 
 Help should be built into your command line so that new users can gradually
@@ -249,9 +266,10 @@ Here are few examples of flag `OPTIONS`:
 
 ## Prompt when input not provided
 
-For **Tier1** commands we want command to improve the discoverability of
-possible input. A new user will most likely not know which `ARGUMENTS` and
-`OPTIONS` are required or which values are possible for those options.
+For *main commands* (as [per classification](#classification)) we want command
+to improve the discoverability of possible input. A new user will most likely
+not know which `ARGUMENTS` and `OPTIONS` are required or which values are
+possible for those options.
 
 In cases, the user might not provide the input or they provide wrong input,
 rather then show the error, prompt a user with an option to find and select
diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md
index e69de29bb..854139a31 100644
--- a/doc/manual/src/contributing/contributing.md
+++ b/doc/manual/src/contributing/contributing.md
@@ -0,0 +1 @@
+# Contributing
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
new file mode 100644
index 000000000..2ad773dea
--- /dev/null
+++ b/doc/manual/src/contributing/hacking.md
@@ -0,0 +1 @@
+# Hacking

From c0f21f08f817745fcf3e9301749b7e237634521c Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 8 Dec 2020 06:57:31 +0100
Subject: [PATCH 552/882] Hide the sqlite statements declarations for the local
 store

These have no need to be in the public interface and it causes spurious
rebuilds each time one wants to add or remove a new statement.
---
 src/libstore/local-store.cc | 76 ++++++++++++++++++++++---------------
 src/libstore/local-store.hh | 15 +-------
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 348e5d0d4..2a47b3956 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -42,6 +42,21 @@
 
 namespace nix {
 
+struct LocalStore::State::Stmts {
+    /* Some precompiled SQLite statements. */
+    SQLiteStmt RegisterValidPath;
+    SQLiteStmt UpdatePathInfo;
+    SQLiteStmt AddReference;
+    SQLiteStmt QueryPathInfo;
+    SQLiteStmt QueryReferences;
+    SQLiteStmt QueryReferrers;
+    SQLiteStmt InvalidatePath;
+    SQLiteStmt AddDerivationOutput;
+    SQLiteStmt QueryValidDerivers;
+    SQLiteStmt QueryDerivationOutputs;
+    SQLiteStmt QueryPathFromHashPart;
+    SQLiteStmt QueryValidPaths;
+};
 
 LocalStore::LocalStore(const Params & params)
     : StoreConfig(params)
@@ -60,6 +75,7 @@ LocalStore::LocalStore(const Params & params)
     , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
 {
     auto state(_state.lock());
+    state->stmts = std::make_unique<State::Stmts>();
 
     /* Create missing state directories if they don't already exist. */
     createDirs(realStoreDir);
@@ -223,31 +239,31 @@ LocalStore::LocalStore(const Params & params)
     else openDB(*state, false);
 
     /* Prepare SQL statements. */
-    state->stmtRegisterValidPath.create(state->db,
+    state->stmts->RegisterValidPath.create(state->db,
         "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
-    state->stmtUpdatePathInfo.create(state->db,
+    state->stmts->UpdatePathInfo.create(state->db,
         "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
-    state->stmtAddReference.create(state->db,
+    state->stmts->AddReference.create(state->db,
         "insert or replace into Refs (referrer, reference) values (?, ?);");
-    state->stmtQueryPathInfo.create(state->db,
+    state->stmts->QueryPathInfo.create(state->db,
         "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
-    state->stmtQueryReferences.create(state->db,
+    state->stmts->QueryReferences.create(state->db,
         "select path from Refs join ValidPaths on reference = id where referrer = ?;");
-    state->stmtQueryReferrers.create(state->db,
+    state->stmts->QueryReferrers.create(state->db,
         "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
-    state->stmtInvalidatePath.create(state->db,
+    state->stmts->InvalidatePath.create(state->db,
         "delete from ValidPaths where path = ?;");
-    state->stmtAddDerivationOutput.create(state->db,
+    state->stmts->AddDerivationOutput.create(state->db,
         "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
-    state->stmtQueryValidDerivers.create(state->db,
+    state->stmts->QueryValidDerivers.create(state->db,
         "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
-    state->stmtQueryDerivationOutputs.create(state->db,
+    state->stmts->QueryDerivationOutputs.create(state->db,
         "select id, path from DerivationOutputs where drv = ?;");
     // Use "path >= ?" with limit 1 rather than "path like '?%'" to
     // ensure efficient lookup.
-    state->stmtQueryPathFromHashPart.create(state->db,
+    state->stmts->QueryPathFromHashPart.create(state->db,
         "select path from ValidPaths where path >= ? limit 1;");
-    state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths");
+    state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
 }
 
 
@@ -590,7 +606,7 @@ void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & out
 void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
 {
     retrySQLite<void>([&]() {
-        state.stmtAddDerivationOutput.use()
+        state.stmts->AddDerivationOutput.use()
             (deriver)
             (outputName)
             (printStorePath(output))
@@ -607,7 +623,7 @@ uint64_t LocalStore::addValidPath(State & state,
         throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
             printStorePath(info.path));
 
-    state.stmtRegisterValidPath.use()
+    state.stmts->RegisterValidPath.use()
         (printStorePath(info.path))
         (info.narHash.to_string(Base16, true))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
@@ -659,7 +675,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
             auto state(_state.lock());
 
             /* Get the path info. */
-            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path)));
+            auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path)));
 
             if (!useQueryPathInfo.next())
                 return std::shared_ptr<ValidPathInfo>();
@@ -679,7 +695,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
 
             info->registrationTime = useQueryPathInfo.getInt(2);
 
-            auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
+            auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3);
             if (s) info->deriver = parseStorePath(s);
 
             /* Note that narSize = NULL yields 0. */
@@ -687,14 +703,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
 
             info->ultimate = useQueryPathInfo.getInt(5) == 1;
 
-            s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
+            s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6);
             if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
-            s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
+            s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7);
             if (s) info->ca = parseContentAddressOpt(s);
 
             /* Get the references. */
-            auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
+            auto useQueryReferences(state->stmts->QueryReferences.use()(info->id));
 
             while (useQueryReferences.next())
                 info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
@@ -709,7 +725,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
 /* Update path info in the database. */
 void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
-    state.stmtUpdatePathInfo.use()
+    state.stmts->UpdatePathInfo.use()
         (info.narSize, info.narSize != 0)
         (info.narHash.to_string(Base16, true))
         (info.ultimate ? 1 : 0, info.ultimate)
@@ -722,7 +738,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 
 uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
 {
-    auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
+    auto use(state.stmts->QueryPathInfo.use()(printStorePath(path)));
     if (!use.next())
         throw InvalidPath("path '%s' is not valid", printStorePath(path));
     return use.getInt(0);
@@ -731,7 +747,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
 
 bool LocalStore::isValidPath_(State & state, const StorePath & path)
 {
-    return state.stmtQueryPathInfo.use()(printStorePath(path)).next();
+    return state.stmts->QueryPathInfo.use()(printStorePath(path)).next();
 }
 
 
@@ -757,7 +773,7 @@ StorePathSet LocalStore::queryAllValidPaths()
 {
     return retrySQLite<StorePathSet>([&]() {
         auto state(_state.lock());
-        auto use(state->stmtQueryValidPaths.use());
+        auto use(state->stmts->QueryValidPaths.use());
         StorePathSet res;
         while (use.next()) res.insert(parseStorePath(use.getStr(0)));
         return res;
@@ -767,7 +783,7 @@ StorePathSet LocalStore::queryAllValidPaths()
 
 void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
 {
-    auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path)));
+    auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path)));
 
     while (useQueryReferrers.next())
         referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
@@ -788,7 +804,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
     return retrySQLite<StorePathSet>([&]() {
         auto state(_state.lock());
 
-        auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path)));
+        auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path)));
 
         StorePathSet derivers;
         while (useQueryValidDerivers.next())
@@ -848,7 +864,7 @@ std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivati
         }
 
         auto useQueryDerivationOutputs {
-            state->stmtQueryDerivationOutputs.use()
+            state->stmts->QueryDerivationOutputs.use()
             (drvId)
         };
 
@@ -872,11 +888,11 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
     return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
         auto state(_state.lock());
 
-        auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
+        auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix));
 
         if (!useQueryPathFromHashPart.next()) return {};
 
-        const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
+        const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0);
         if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
             return parseStorePath(s);
         return {};
@@ -990,7 +1006,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         for (auto & [_, i] : infos) {
             auto referrer = queryValidPathId(*state, i.path);
             for (auto & j : i.references)
-                state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
+                state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
         }
 
         /* Check that the derivation outputs are correct.  We can't do
@@ -1030,7 +1046,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 {
     debug("invalidating path '%s'", printStorePath(path));
 
-    state.stmtInvalidatePath.use()(printStorePath(path)).exec();
+    state.stmts->InvalidatePath.use()(printStorePath(path)).exec();
 
     /* Note that the foreign key constraints on the Refs table take
        care of deleting the references entries for `path'. */
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 58ec93f27..332718af4 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -55,19 +55,8 @@ private:
         /* The SQLite database object. */
         SQLite db;
 
-        /* Some precompiled SQLite statements. */
-        SQLiteStmt stmtRegisterValidPath;
-        SQLiteStmt stmtUpdatePathInfo;
-        SQLiteStmt stmtAddReference;
-        SQLiteStmt stmtQueryPathInfo;
-        SQLiteStmt stmtQueryReferences;
-        SQLiteStmt stmtQueryReferrers;
-        SQLiteStmt stmtInvalidatePath;
-        SQLiteStmt stmtAddDerivationOutput;
-        SQLiteStmt stmtQueryValidDerivers;
-        SQLiteStmt stmtQueryDerivationOutputs;
-        SQLiteStmt stmtQueryPathFromHashPart;
-        SQLiteStmt stmtQueryValidPaths;
+        struct Stmts;
+        std::unique_ptr<Stmts> stmts;
 
         /* The file to which we write our temporary roots. */
         AutoCloseFD fdTempRoots;

From 6758e65612b990805d3d7d2039cd92647730e900 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Dec 2020 09:44:07 +0100
Subject: [PATCH 553/882] Revert "Re-query for the derivation outputs in the
 post-build-hook"

This reverts commit 1b1e0760335832c87516b9103b670b34662d5daf.

Using `queryPartialDerivationOutputMap` assumes that the derivation
exists locally which isn't the case for remote builders.
---
 src/libstore/build/derivation-goal.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index fdf777c27..1db85bd37 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -899,10 +899,8 @@ void DerivationGoal::buildDone()
                 Logger::Fields{worker.store.printStorePath(drvPath)});
             PushActivity pact(act.id);
             StorePathSet outputPaths;
-            for (auto& [_, maybeOutPath] :
-                 worker.store.queryPartialDerivationOutputMap(drvPath)) {
-                if (maybeOutPath)
-                    outputPaths.insert(*maybeOutPath);
+            for (auto i : drv->outputs) {
+                outputPaths.insert(finalOutputs.at(i.first));
             }
             std::map<std::string, std::string> hookEnvironment = getEnv();
 

From ee7c94fa1b74b43f7a719373f5f566d09b147126 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Dec 2020 10:37:39 +0100
Subject: [PATCH 554/882] Test the post-build-hook with remote builders

Regression test for #4245
---
 tests/build-remote-input-addressed.sh | 28 +++++++++++++++++++++++++++
 tests/build-remote.sh                 |  3 +++
 2 files changed, 31 insertions(+)

diff --git a/tests/build-remote-input-addressed.sh b/tests/build-remote-input-addressed.sh
index b34caa061..49d15c389 100644
--- a/tests/build-remote-input-addressed.sh
+++ b/tests/build-remote-input-addressed.sh
@@ -3,3 +3,31 @@ source common.sh
 file=build-hook.nix
 
 source build-remote.sh
+
+# Add a `post-build-hook` option to the nix conf.
+# This hook will be executed both for the local machine and the remote builders
+# (because they share the same config).
+registerBuildHook () {
+    # Dummy post-build-hook just to ensure that it's executed correctly.
+    # (we can't reuse the one from `$PWD/push-to-store.sh` because of
+    # https://github.com/NixOS/nix/issues/4341)
+    cat <<EOF > $TEST_ROOT/post-build-hook.sh
+#!/bin/sh
+
+echo "Post hook ran successfully"
+# Add an empty line to a counter file, just to check that this hook ran properly
+echo "" >> $TEST_ROOT/post-hook-counter
+EOF
+    chmod +x $TEST_ROOT/post-build-hook.sh
+    rm -f $TEST_ROOT/post-hook-counter
+
+    echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf
+}
+
+registerBuildHook
+source build-remote.sh
+
+# `build-hook.nix` has four derivations to build, and the hook runs twice for
+# each derivation (once on the builder and once on the host), so the counter
+# should contain eight lines now
+[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]]
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index ca6d1de09..04848e4b5 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -14,6 +14,9 @@ builders=(
   "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
 )
 
+chmod -R +w $TEST_ROOT/machine* || true
+rm -rf $TEST_ROOT/machine* || true
+
 # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
 # child process. This allows us to test LegacySSHStore::buildDerivation().
 # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().

From c87267c2a4621a42d6bfcdb7944cd546f194787a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Dec 2020 10:38:04 +0100
Subject: [PATCH 555/882] Store the final drv outputs in memory when building
 remotely
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The `DerivationGoal` has a variable storing the “final” derivation
output paths that is used (amongst other things) to fill the environment
for the post build hook. However this variable wasn't set when the
build-hook is used, causing a crash when both hooks are used together.

Fix this by setting this variable (from the informations in the db) after a run
of the post build hook.
---
 src/libstore/build/derivation-goal.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 1db85bd37..de58d9f06 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -2872,6 +2872,8 @@ void DerivationGoal::registerOutputs()
         for (auto & i : drv->outputsAndOptPaths(worker.store)) {
             if (!i.second.second || !worker.store.isValidPath(*i.second.second))
                 allValid = false;
+            else
+                finalOutputs.insert_or_assign(i.first, *i.second.second);
         }
         if (allValid) return;
     }

From 93a8a005defbe5204f739853403ef11dbc016f33 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 5 Dec 2020 15:33:16 +0100
Subject: [PATCH 556/882] libstore/openStore: fix stores with IPv6 addresses

In `nixStable` (2.3.7 to be precise) it's possible to connect to stores
using an IPv6 address:

  nix ping-store --store ssh://root@2001:db8::1

This is also useful for `nixops(1)` where you could specify an IPv6
address in `deployment.targetHost`.

However, this behavior is broken on `nixUnstable` and fails with the
following error:

  $ nix store ping --store ssh://root@2001:db8::1
  don't know how to open Nix store 'ssh://root@2001:db8::1'

This happened because `openStore` from `libstore` uses the `parseURL`
function from `libfetchers` which expects a valid URL as defined in
RFC2732. However, this is unsupported by `ssh(1)`:

  $ nix store ping --store 'ssh://root@[2001:db8::1]'
  cannot connect to 'root@[2001:db8::1]'

This patch now allows both ways of specifying a store (`root@2001:db8::1`) and
also `root@[2001:db8::1]` since the latter one is useful to pass query
parameters to the remote store.

In order to achieve this, the following changes were made:

* The URL regex from `url-parts.hh` now allows an IPv6 address in the
  form `2001:db8::1` and also `[2001:db8::1]`.

* In `libstore`, a new function named `extractConnStr` ensures that a
  proper URL is passed to e.g. `ssh(1)`:

  * If a URL looks like either `[2001:db8::1]` or `root@[2001:db8::1]`,
    the brackets will be removed using a regex. No additional validation
    is done here as only strings parsed by `parseURL` are expected.

  * In any other case, the string will be left untouched.

* The rules above only apply for `LegacySSHStore` and `SSHStore` (a.k.a
  `ssh://` and `ssh-ng://`).

Unresolved questions:

* I'm not really sure whether we want to allow both variants of IPv6
  addresses in the URL parser. However it should be noted that both seem
  to be possible according to RFC2732:

  > This document incudes an update to the generic syntax for Uniform
  > Resource Identifiers defined in RFC 2396 [URL].  It defines a syntax
  > for IPv6 addresses and allows the use of "[" and "]" within a URI
  > explicitly for this reserved purpose.

* Currently, it's not supported to specify a port number behind the
  hostname, however it seems as this is not really supported by the URL
  parser. Hence, this is probably out of scope here.
---
 src/libstore/store-api.cc | 35 ++++++++++++++++++++++++++++++++++-
 src/libutil/url-parts.hh  |  3 ++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 27be66cac..7bf9235b2 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -10,6 +10,8 @@
 #include "archive.hh"
 #include "callback.hh"
 
+#include <regex>
+
 namespace nix {
 
 
@@ -1091,6 +1093,34 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
     }
 }
 
+// The `parseURL` function supports both IPv6 URIs as defined in
+// RFC2732, but also pure addresses. The latter one is needed here to
+// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
+//
+// This function now ensures that a usable connection string is available:
+// * If the store to be opened is not an SSH store, nothing will be done.
+// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
+//   needed to pass further flags), it
+//   will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
+// * If the URL looks like `root@::1` it will be left as-is.
+// * In any other case, the string will be left as-is.
+static std::string extractConnStr(const std::string &proto, const std::string &connStr)
+{
+    if (proto.rfind("ssh") != std::string::npos) {
+        std::smatch result;
+        std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
+
+        if (std::regex_match(connStr, result, v6AddrRegex)) {
+            if (result[1].matched) {
+                return result.str(1) + result.str(3);
+            }
+            return result.str(3);
+        }
+    }
+
+    return connStr;
+}
+
 ref<Store> openStore(const std::string & uri_,
     const Store::Params & extraParams)
 {
@@ -1099,7 +1129,10 @@ ref<Store> openStore(const std::string & uri_,
         auto parsedUri = parseURL(uri_);
         params.insert(parsedUri.query.begin(), parsedUri.query.end());
 
-        auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
+        auto baseURI = extractConnStr(
+            parsedUri.scheme,
+            parsedUri.authority.value_or("") + parsedUri.path
+        );
 
         for (auto implem : *Implementations::registered) {
             if (implem.uriSchemes.count(parsedUri.scheme)) {
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 68be15cb0..5d21b8d1a 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -8,7 +8,8 @@ namespace nix {
 // URI stuff.
 const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
 const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
-const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
+const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+";
+const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
 const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
 const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
 const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";

From 5286310e593b2ac13abbbac90c5b458c4be2ad37 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Dec 2020 14:53:45 +0100
Subject: [PATCH 557/882] Use no substituers by default in the tests

Otherwise https://cache.nixos.org is chosen by default, causing the OSX
testsuite to hang inside the sandbox.

(In a way, this is probably rugging an actual bug under the carpet as
Nix should be able to gracefully timeout in such a case, but that's
beyond mac OSX-fu)
---
 tests/init.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/init.sh b/tests/init.sh
index f9ced6b0d..63cf895e2 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -19,6 +19,7 @@ keep-derivations = false
 sandbox = false
 experimental-features = nix-command flakes
 gc-reserved-space = 0
+substituters =
 flake-registry = $TEST_ROOT/registry.json
 include nix.conf.extra
 EOF

From a8f533b66417a1025a468cae3068bd2f5c06e811 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Nov 2020 11:19:36 +0100
Subject: [PATCH 558/882] Add lvlNotice log level

This is like syslog's LOG_NOTICE: "normal, but significant,
condition".
---
 src/libutil/error.hh   | 1 +
 src/libutil/logging.hh | 1 +
 src/nix/main.cc        | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index d1b6d82bb..aa4fadfcc 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -45,6 +45,7 @@ namespace nix {
 typedef enum {
     lvlError = 0,
     lvlWarn,
+    lvlNotice,
     lvlInfo,
     lvlTalkative,
     lvlChatty,
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 82ba54051..96ad69790 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -198,6 +198,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
     } while (0)
 
 #define printError(args...) printMsg(lvlError, args)
+#define notice(args...) printMsg(lvlNotice, args)
 #define printInfo(args...) printMsg(lvlInfo, args)
 #define printTalkative(args...) printMsg(lvlTalkative, args)
 #define debug(args...) printMsg(lvlDebug, args)
diff --git a/src/nix/main.cc b/src/nix/main.cc
index fb3bffeaf..27b1d7257 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -250,7 +250,7 @@ void mainWrapped(int argc, char * * argv)
         if (legacy) return legacy(argc, argv);
     }
 
-    verbosity = lvlWarn;
+    verbosity = lvlNotice;
     settings.verboseBuild = false;
     evalSettings.pureEval = true;
 

From c6a1bcd0ec1ed443947ae7151e32dd6827dfe53e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 17:11:39 +0100
Subject: [PATCH 559/882] nix store make-content-addressable: Show rewritten
 path

---
 src/nix/make-content-addressable.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 0dade90ef..5165c4804 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             };
 
             if (!json)
-                printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
+                notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
 
             auto source = sinkToSource([&](Sink & nextSink) {
                 RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);

From eb453081092cbee5f8176c1d348ac23e46a24281 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 10 Dec 2020 17:40:00 +0100
Subject: [PATCH 560/882] Fix the `nix` command with CA derivations

Prevents a crash because most `nix` subcommands assumed that derivations
know their output path, which isn't the case for CA derivations
---
 src/nix/installables.cc    | 2 +-
 tests/content-addressed.sh | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index b6ed030af..3506c3fcc 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -409,7 +409,7 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
     for (auto & drvInfo : drvInfos) {
         res.push_back({
             state->store->parseStorePath(drvInfo.queryDrvPath()),
-            state->store->parseStorePath(drvInfo.queryOutPath()),
+            state->store->maybeParseStorePath(drvInfo.queryOutPath()),
             drvInfo.queryOutputName()
         });
     }
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index 03eff549c..bc37a99c1 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -50,7 +50,13 @@ testGC () {
     nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
 }
 
+testNixCommand () {
+    clearStore
+    nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link
+}
+
 testRemoteCache
 testDeterministicCA
 testCutoff
 testGC
+testNixCommand

From 63b3536f50f124cdcd7592b344eac157a1439d42 Mon Sep 17 00:00:00 2001
From: Michael Bishop <cleverca22@gmail.com>
Date: Fri, 28 Aug 2020 10:28:35 -0300
Subject: [PATCH 561/882] treat s3 permission errors as file-not-found

Signed-off-by: Jonathan Ringer <jonringer117@gmail.com>
---
 src/libstore/s3-binary-cache-store.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 27253fc12..d6edafd7e 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -166,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject(
             dynamic_cast<std::stringstream &>(result.GetBody()).str());
 
     } catch (S3Error & e) {
-        if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
+        if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) &&
+            (e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw;
     }
 
     auto now2 = std::chrono::steady_clock::now();

From 58cdab64acd4807f73768fb32acdde39b501799f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 8 Oct 2020 17:36:51 +0200
Subject: [PATCH 562/882] Store metadata about drv outputs realisations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For each known realisation, store:
- its output
- its output path

This comes with a set of needed changes:

- New `realisations` module declaring the types needed for describing
  these mappings
- New `Store::registerDrvOutput` method registering all the needed informations
  about a derivation output (also replaces `LocalStore::linkDeriverToPath`)
- new `Store::queryRealisation` method to retrieve the informations for a
  derivations

This introcudes some redundancy on the remote-store side between
`wopQueryDerivationOutputMap` and `wopQueryRealisation`.
However we might need to keep both (regardless of backwards compat)
because we sometimes need to get some infos for all the outputs of a
derivation (where `wopQueryDerivationOutputMap` is handy), but all the
stores can't implement it − because listing all the outputs of a
derivation isn't really possible for binary caches where the server
doesn't allow to list a directory.
---
 src/libstore/binary-cache-store.cc       | 17 ++++++
 src/libstore/binary-cache-store.hh       |  7 +++
 src/libstore/build/derivation-goal.cc    | 19 ++++++-
 src/libstore/daemon.cc                   | 22 ++++++++
 src/libstore/dummy-store.cc              |  3 +
 src/libstore/legacy-ssh-store.cc         |  4 ++
 src/libstore/local-binary-cache-store.cc |  1 +
 src/libstore/local-store.cc              | 21 +++++--
 src/libstore/local-store.hh              | 12 ++--
 src/libstore/realisation.cc              | 72 ++++++++++++++++++++++++
 src/libstore/realisation.hh              | 34 +++++++++++
 src/libstore/remote-store.cc             | 21 +++++++
 src/libstore/remote-store.hh             |  4 ++
 src/libstore/store-api.hh                | 15 +++++
 src/libstore/worker-protocol.hh          |  5 ++
 tests/content-addressed.sh               |  3 +-
 16 files changed, 248 insertions(+), 12 deletions(-)
 create mode 100644 src/libstore/realisation.cc
 create mode 100644 src/libstore/realisation.hh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index a918b7208..085dc7ba1 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -443,6 +443,23 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     })->path;
 }
 
+std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
+{
+    auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
+    auto rawOutputInfo = getFile(outputInfoFilePath);
+
+    if (rawOutputInfo) {
+        return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) };
+    } else {
+        return std::nullopt;
+    }
+}
+
+void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
+    auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
+    upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath");
+}
+
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
 {
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 5224d7ec8..07a8b2beb 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -33,6 +33,9 @@ private:
 
 protected:
 
+    // The prefix under which realisation infos will be stored
+    const std::string realisationsPrefix = "/realisations";
+
     BinaryCacheStore(const Params & params);
 
 public:
@@ -99,6 +102,10 @@ public:
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
 
+    void registerDrvOutput(const Realisation & info) override;
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
+
     void narFromPath(const StorePath & path, Sink & sink) override;
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index de58d9f06..a0f10c33d 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -2094,6 +2094,20 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
         /* Nothing to be done; 'path' must already be valid. */
     }
 
+    void registerDrvOutput(const Realisation & info) override
+    {
+        if (!goal.isAllowed(info.id.drvPath))
+            throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath));
+        next->registerDrvOutput(info);
+    }
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
+    {
+        if (!goal.isAllowed(id.drvPath))
+            throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath));
+        return next->queryRealisation(id);
+    }
+
     void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
@@ -3393,7 +3407,10 @@ void DerivationGoal::registerOutputs()
 
     if (useDerivation || isCaFloating)
         for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
+            worker.store.registerDrvOutput(
+                DrvOutputId{drvPathResolved, outputName},
+                DrvOutputInfo{.outPath = newInfo.path,
+                              .resolvedDrv = drvPathResolved});
 }
 
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 2224d54d5..ba5788b64 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopRegisterDrvOutput: {
+        logger->startWork();
+        auto outputId = DrvOutput::parse(readString(from));
+        auto outputPath = StorePath(readString(from));
+        auto resolvedDrv = StorePath(readString(from));
+        store->registerDrvOutput(Realisation{
+            .id = outputId, .outPath = outputPath});
+        logger->stopWork();
+        break;
+    }
+
+    case wopQueryRealisation: {
+        logger->startWork();
+        auto outputId = DrvOutput::parse(readString(from));
+        auto info = store->queryRealisation(outputId);
+        logger->stopWork();
+        std::set<StorePath> outPaths;
+        if (info) outPaths.insert(info->outPath);
+        worker_proto::write(*store, to, outPaths);
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 98b745c3a..91fc178db 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     { unsupported("buildDerivation"); }
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput&) override
+    { unsupported("queryRealisation"); }
 };
 
 static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 467169ce8..ad1779aea 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -333,6 +333,10 @@ public:
         auto conn(connections->get());
         return conn->remoteVersion;
     }
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput&) override
+    // TODO: Implement
+    { unsupported("queryRealisation"); }
 };
 
 static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 7d979c5c2..bb7464989 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -87,6 +87,7 @@ protected:
 void LocalBinaryCacheStore::init()
 {
     createDirs(binaryCacheDir + "/nar");
+    createDirs(binaryCacheDir + realisationsPrefix);
     if (writeDebugInfo)
         createDirs(binaryCacheDir + "/debuginfo");
     BinaryCacheStore::init();
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2a47b3956..418b3ab9c 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -597,13 +597,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 }
 
 
-void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
+void LocalStore::registerDrvOutput(const Realisation & info)
 {
     auto state(_state.lock());
-    return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
+    // XXX: This ignores the references of the output because we can
+    // recompute them later from the drv and the references of the associated
+    // store path, but doing so is both inefficient and fragile.
+    return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath);
 }
 
-void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
+void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
 {
     retrySQLite<void>([&]() {
         state.stmts->AddDerivationOutput.use()
@@ -653,7 +656,7 @@ uint64_t LocalStore::addValidPath(State & state,
             /* Floating CA derivations have indeterminate output paths until
                they are built, so don't register anything in that case */
             if (i.second.second)
-                linkDeriverToPath(state, id, i.first, *i.second.second);
+                registerDrvOutput_(state, id, i.first, *i.second.second);
         }
     }
 
@@ -1612,5 +1615,13 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
     }
 }
 
-
+std::optional<const DrvOutputInfo> LocalStore::queryDrvOutputInfo(const DrvOutputId& id) {
+    auto outputPath = queryOutputPathOf(id.drvPath, id.outputName);
+    if (!(outputPath && isValidPath(*outputPath)))
+        return std::nullopt;
+    else
+        return {DrvOutputInfo{
+            .outPath = *outputPath,
+        }};
 }
+}  // namespace nix
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 332718af4..440411f01 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -208,6 +208,13 @@ public:
        garbage until it exceeds maxFree. */
     void autoGC(bool sync = true);
 
+    /* Register the store path 'output' as the output named 'outputName' of
+       derivation 'deriver'. */
+    void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override;
+    void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
+
 private:
 
     int getSchema();
@@ -276,11 +283,6 @@ private:
        specified by the ‘secret-key-files’ option. */
     void signPathInfo(ValidPathInfo & info);
 
-    /* Register the store path 'output' as the output named 'outputName' of
-       derivation 'deriver'. */
-    void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
-    void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
-
     Path getRealStoreDir() override { return realStoreDir; }
 
     void createUser(const std::string & userName, uid_t userId) override;
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
new file mode 100644
index 000000000..fcc1a3825
--- /dev/null
+++ b/src/libstore/realisation.cc
@@ -0,0 +1,72 @@
+#include "realisation.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+MakeError(InvalidDerivationOutputId, Error);
+
+DrvOutput DrvOutput::parse(const std::string &strRep) {
+    const auto &[rawPath, outputs] = parsePathWithOutputs(strRep);
+    if (outputs.size() != 1)
+        throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
+
+    return DrvOutput{
+        .drvPath = StorePath(rawPath),
+        .outputName = *outputs.begin(),
+    };
+}
+
+std::string DrvOutput::to_string() const {
+    return std::string(drvPath.to_string()) + "!" + outputName;
+}
+
+std::string Realisation::to_string() const {
+    std::string res;
+
+    res += "Id: " + id.to_string() + '\n';
+    res += "OutPath: " + std::string(outPath.to_string()) + '\n';
+
+    return res;
+}
+
+Realisation Realisation::parse(const std::string & s, const std::string & whence)
+{
+    // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out
+    auto corrupt = [&]() {
+        return Error("Drv output info file '%1%' is corrupt", whence);
+    };
+
+    std::optional<DrvOutput> id;
+    std::optional<StorePath> outPath;
+
+    size_t pos = 0;
+    while (pos < s.size()) {
+
+        size_t colon = s.find(':', pos);
+        if (colon == std::string::npos) throw corrupt();
+
+        std::string name(s, pos, colon - pos);
+
+        size_t eol = s.find('\n', colon + 2);
+        if (eol == std::string::npos) throw corrupt();
+
+        std::string value(s, colon + 2, eol - colon - 2);
+
+        if (name == "Id")
+            id = DrvOutput::parse(value);
+
+        if (name == "OutPath")
+            outPath = StorePath(value);
+
+        pos = eol + 1;
+    }
+
+    if (!outPath) corrupt();
+    if (!id) corrupt();
+    return Realisation {
+        .id = *id,
+        .outPath = *outPath,
+    };
+}
+
+} // namespace nix
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
new file mode 100644
index 000000000..c573e1bb4
--- /dev/null
+++ b/src/libstore/realisation.hh
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "path.hh"
+
+namespace nix {
+
+struct DrvOutput {
+    StorePath drvPath;
+    std::string outputName;
+
+    std::string to_string() const;
+
+    static DrvOutput parse(const std::string &);
+
+    bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
+    bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
+
+private:
+    // Just to make comparison operators easier to write
+    std::pair<StorePath, std::string> to_pair() const
+    { return std::make_pair(drvPath, outputName); }
+};
+
+struct Realisation {
+    DrvOutput id;
+    StorePath outPath;
+
+    std::string to_string() const;
+    static Realisation parse(const std::string & s, const std::string & whence);
+};
+
+typedef std::map<DrvOutput, Realisation> DrvOutputs;
+
+}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index be29f8e6f..f1f4d0516 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
     return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
 }
 
+void RemoteStore::registerDrvOutput(const Realisation & info)
+{
+    auto conn(getConnection());
+    conn->to << wopRegisterDrvOutput;
+    conn->to << info.id.to_string();
+    conn->to << std::string(info.outPath.to_string());
+    conn.processStderr();
+}
+
+std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
+{
+    auto conn(getConnection());
+    conn->to << wopQueryRealisation;
+    conn->to << id.to_string();
+    conn.processStderr();
+    auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
+    if (outPaths.empty())
+        return std::nullopt;
+    return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+}
+
 
 void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
 {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 9f78fcb02..fdd53e6ed 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -81,6 +81,10 @@ public:
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
 
+    void registerDrvOutput(const Realisation & info) override;
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
+
     void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6b9331495..7cdadc1f3 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "realisation.hh"
 #include "path.hh"
 #include "hash.hh"
 #include "content-address.hh"
@@ -396,6 +397,8 @@ protected:
 
 public:
 
+    virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
+
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
     virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
@@ -468,6 +471,18 @@ public:
     virtual StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
 
+    /**
+     * Add a mapping indicating that `deriver!outputName` maps to the output path
+     * `output`.
+     *
+     * This is redundant for known-input-addressed and fixed-output derivations
+     * as this information is already present in the drv file, but necessary for
+     * floating-ca derivations and their dependencies as there's no way to
+     * retrieve this information otherwise.
+     */
+    virtual void registerDrvOutput(const Realisation & output)
+    { unsupported("registerDrvOutput"); }
+
     /* Write a NAR dump of a store path. */
     virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 63bd6ea49..f2cdc7ca3 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -1,5 +1,8 @@
 #pragma once
 
+#include "store-api.hh"
+#include "serialise.hh"
+
 namespace nix {
 
 
@@ -50,6 +53,8 @@ typedef enum {
     wopAddToStoreNar = 39,
     wopQueryMissing = 40,
     wopQueryDerivationOutputMap = 41,
+    wopRegisterDrvOutput = 42,
+    wopQueryRealisation = 43,
 } WorkerOp;
 
 
diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index bc37a99c1..e8ac88609 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -55,7 +55,8 @@ testNixCommand () {
     nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link
 }
 
-testRemoteCache
+# Disabled until we have it properly working
+# testRemoteCache
 testDeterministicCA
 testCutoff
 testGC

From 3ac9d74eb1de0f696bb0384132f7ecc7d057f9d6 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 20 Oct 2020 15:14:02 +0200
Subject: [PATCH 563/882] Rework the db schema for derivation outputs

Add a new table for tracking the derivation output mappings.

We used to hijack the `DerivationOutputs` table for that, but (despite its
name), it isn't a really good fit:

- Its entries depend on the drv being a valid path, making it play badly with
  garbage collection and preventing us to copy a drv output without copying
  the whole drv closure too;
- It dosen't guaranty that the output path exists;

By using a different table, we can experiment with a different schema better
suited for tracking the output mappings of CA derivations.
(incidentally, this also fixes #4138)
---
 src/libstore/build/derivation-goal.cc |  16 +-
 src/libstore/ca-specific-schema.sql   |  11 ++
 src/libstore/local-store.cc           | 217 ++++++++++++++++++--------
 src/libstore/local-store.hh           |   4 +-
 src/libstore/local.mk                 |   4 +-
 5 files changed, 172 insertions(+), 80 deletions(-)
 create mode 100644 src/libstore/ca-specific-schema.sql

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index a0f10c33d..b7bf866eb 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -493,8 +493,9 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
-            || fullDrv.type() == DerivationType::DeferredInputAddressed) {
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+            ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
+            || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
             /* We are be able to resolve this derivation based on the
                now-known results of dependencies. If so, we become a stub goal
                aliasing that resolved derivation goal */
@@ -3405,12 +3406,11 @@ void DerivationGoal::registerOutputs()
         drvPathResolved = writeDerivation(worker.store, drv2);
     }
 
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.registerDrvOutput(
-                DrvOutputId{drvPathResolved, outputName},
-                DrvOutputInfo{.outPath = newInfo.path,
-                              .resolvedDrv = drvPathResolved});
+    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+        for (auto& [outputName, newInfo] : infos)
+            worker.store.registerDrvOutput(Realisation{
+                .id = DrvOutput{drvPathResolved, outputName},
+                .outPath = newInfo.path});
 }
 
 
diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql
new file mode 100644
index 000000000..93c442826
--- /dev/null
+++ b/src/libstore/ca-specific-schema.sql
@@ -0,0 +1,11 @@
+-- Extension of the sql schema for content-addressed derivations.
+-- Won't be loaded unless the experimental feature `ca-derivations`
+-- is enabled
+
+create table if not exists Realisations (
+    drvPath text not null,
+    outputName text not null, -- symbolic output id, usually "out"
+    outputPath integer not null,
+    primary key (drvPath, outputName),
+    foreign key (outputPath) references ValidPaths(id) on delete cascade
+);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 418b3ab9c..69ab821d9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -52,12 +52,52 @@ struct LocalStore::State::Stmts {
     SQLiteStmt QueryReferrers;
     SQLiteStmt InvalidatePath;
     SQLiteStmt AddDerivationOutput;
+    SQLiteStmt RegisterRealisedOutput;
     SQLiteStmt QueryValidDerivers;
     SQLiteStmt QueryDerivationOutputs;
+    SQLiteStmt QueryRealisedOutput;
+    SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
 };
 
+int getSchema(Path schemaPath)
+{
+    int curSchema = 0;
+    if (pathExists(schemaPath)) {
+        string s = readFile(schemaPath);
+        if (!string2Int(s, curSchema))
+            throw Error("'%1%' is corrupt", schemaPath);
+    }
+    return curSchema;
+}
+
+void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
+{
+    const int nixCASchemaVersion = 1;
+    int curCASchema = getSchema(schemaPath);
+    if (curCASchema != nixCASchemaVersion) {
+        if (curCASchema > nixCASchemaVersion) {
+            throw Error("current Nix store ca-schema is version %1%, but I only support %2%",
+                 curCASchema, nixCASchemaVersion);
+        }
+
+        if (!lockFile(lockFd.get(), ltWrite, false)) {
+            printInfo("waiting for exclusive access to the Nix store for ca drvs...");
+            lockFile(lockFd.get(), ltWrite, true);
+        }
+
+        if (curCASchema == 0) {
+            static const char schema[] =
+              #include "ca-specific-schema.sql.gen.hh"
+                ;
+            db.exec(schema);
+        }
+        writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
+        lockFile(lockFd.get(), ltRead, true);
+    }
+}
+
 LocalStore::LocalStore(const Params & params)
     : StoreConfig(params)
     , Store(params)
@@ -238,6 +278,10 @@ LocalStore::LocalStore(const Params & params)
 
     else openDB(*state, false);
 
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
+    }
+
     /* Prepare SQL statements. */
     state->stmts->RegisterValidPath.create(state->db,
         "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
@@ -264,6 +308,28 @@ LocalStore::LocalStore(const Params & params)
     state->stmts->QueryPathFromHashPart.create(state->db,
         "select path from ValidPaths where path >= ? limit 1;");
     state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        state->stmts->RegisterRealisedOutput.create(state->db,
+            R"(
+                insert or replace into Realisations (drvPath, outputName, outputPath)
+                values (?, ?, (select id from ValidPaths where path = ?))
+                ;
+            )");
+        state->stmts->QueryRealisedOutput.create(state->db,
+            R"(
+                select Output.path from Realisations
+                    inner join ValidPaths as Output on Output.id = Realisations.outputPath
+                    where drvPath = ? and outputName = ?
+                    ;
+            )");
+        state->stmts->QueryAllRealisedOutputs.create(state->db,
+            R"(
+                select outputName, Output.path from Realisations
+                    inner join ValidPaths as Output on Output.id = Realisations.outputPath
+                    where drvPath = ?
+                    ;
+            )");
+    }
 }
 
 
@@ -301,16 +367,7 @@ std::string LocalStore::getUri()
 
 
 int LocalStore::getSchema()
-{
-    int curSchema = 0;
-    if (pathExists(schemaPath)) {
-        string s = readFile(schemaPath);
-        if (!string2Int(s, curSchema))
-            throw Error("'%1%' is corrupt", schemaPath);
-    }
-    return curSchema;
-}
-
+{ return nix::getSchema(schemaPath); }
 
 void LocalStore::openDB(State & state, bool create)
 {
@@ -600,13 +657,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
     auto state(_state.lock());
-    // XXX: This ignores the references of the output because we can
-    // recompute them later from the drv and the references of the associated
-    // store path, but doing so is both inefficient and fragile.
-    return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath);
+    retrySQLite<void>([&]() {
+        state->stmts->RegisterRealisedOutput.use()
+            (info.id.drvPath.to_string())
+            (info.id.outputName)
+            (printStorePath(info.outPath))
+            .exec();
+    });
 }
 
-void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
+void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
 {
     retrySQLite<void>([&]() {
         state.stmts->AddDerivationOutput.use()
@@ -656,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state,
             /* Floating CA derivations have indeterminate output paths until
                they are built, so don't register anything in that case */
             if (i.second.second)
-                registerDrvOutput_(state, id, i.first, *i.second.second);
+                cacheDrvOutputMapping(state, id, i.first, *i.second.second);
         }
     }
 
@@ -817,70 +877,85 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
     });
 }
 
-
-std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
+// Try to resolve the derivation at path `original`, with a caching layer
+// to make it more efficient
+std::optional<StorePath> cachedResolve(
+    LocalStore & store,
+    const StorePath & original)
 {
-    auto path = path_;
-    std::map<std::string, std::optional<StorePath>> outputs;
-    Derivation drv = readDerivation(path);
-    for (auto & [outName, _] : drv.outputs) {
-        outputs.insert_or_assign(outName, std::nullopt);
-    }
-    bool haveCached = false;
     {
         auto resolutions = drvPathResolutions.lock();
-        auto resolvedPathOptIter = resolutions->find(path);
+        auto resolvedPathOptIter = resolutions->find(original);
         if (resolvedPathOptIter != resolutions->end()) {
             auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
             if (resolvedPathOpt)
-                path = *resolvedPathOpt;
-            haveCached = true;
+                return resolvedPathOpt;
         }
     }
-    /* can't just use else-if instead of `!haveCached` because we need to unlock
-       `drvPathResolutions` before it is locked in `Derivation::resolve`. */
-    if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) {
-        /* Try resolve drv and use that path instead. */
-        auto attempt = drv.tryResolve(*this);
-        if (!attempt)
-            /* If we cannot resolve the derivation, we cannot have any path
-               assigned so we return the map of all std::nullopts. */
-            return outputs;
-        /* Just compute store path */
-        auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
-        /* Store in memo table. */
-        /* FIXME: memo logic should not be local-store specific, should have
-           wrapper-method instead. */
-        drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
-        path = std::move(pathResolved);
-    }
-    return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
-        auto state(_state.lock());
 
+    /* Try resolve drv and use that path instead. */
+    auto drv = store.readDerivation(original);
+    auto attempt = drv.tryResolve(store);
+    if (!attempt)
+        return std::nullopt;
+    /* Just compute store path */
+    auto pathResolved =
+        writeDerivation(store, *std::move(attempt), NoRepair, true);
+    /* Store in memo table. */
+    drvPathResolutions.lock()->insert_or_assign(original, pathResolved);
+    return pathResolved;
+}
+
+std::map<std::string, std::optional<StorePath>>
+LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
+{
+    auto path = path_;
+    auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
+        auto state(_state.lock());
+        std::map<std::string, std::optional<StorePath>> outputs;
         uint64_t drvId;
         try {
             drvId = queryValidPathId(*state, path);
-        } catch (InvalidPath &) {
-            /* FIXME? if the derivation doesn't exist, we cannot have a mapping
-               for it. */
-            return outputs;
+        } catch (InvalidPath&) {
+            // Ignore non-existing drvs as they might still have an output map
+            // defined if ca-derivations is enabled
         }
+        auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
+        while (use.next())
+            outputs.insert_or_assign(
+                use.getStr(0), parseStorePath(use.getStr(1)));
 
-        auto useQueryDerivationOutputs {
-            state->stmts->QueryDerivationOutputs.use()
-            (drvId)
-        };
+        return outputs;
+    });
+
+    if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
+        return outputs;
+
+    auto drv = readDerivation(path);
+
+    for (auto & output : drv.outputsAndOptPaths(*this)) {
+        outputs.emplace(output.first, std::nullopt);
+    }
+
+    auto resolvedDrv = cachedResolve(*this, path);
+
+    if (!resolvedDrv)
+        return outputs;
+
+    retrySQLite<void>([&]() {
+        auto state(_state.lock());
+        path = *resolvedDrv;
+        auto useQueryDerivationOutputs{
+            state->stmts->QueryAllRealisedOutputs.use()(path.to_string())};
 
         while (useQueryDerivationOutputs.next())
             outputs.insert_or_assign(
                 useQueryDerivationOutputs.getStr(0),
-                parseStorePath(useQueryDerivationOutputs.getStr(1))
-            );
-
-        return outputs;
+                parseStorePath(useQueryDerivationOutputs.getStr(1)));
     });
-}
 
+    return outputs;
+}
 
 std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
 {
@@ -1615,13 +1690,19 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
     }
 }
 
-std::optional<const DrvOutputInfo> LocalStore::queryDrvOutputInfo(const DrvOutputId& id) {
-    auto outputPath = queryOutputPathOf(id.drvPath, id.outputName);
-    if (!(outputPath && isValidPath(*outputPath)))
-        return std::nullopt;
-    else
-        return {DrvOutputInfo{
-            .outPath = *outputPath,
-        }};
+std::optional<const Realisation> LocalStore::queryRealisation(
+    const DrvOutput& id) {
+    typedef std::optional<const Realisation> Ret;
+    return retrySQLite<Ret>([&]() -> Ret {
+        auto state(_state.lock());
+        auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(
+            id.outputName));
+        if (!use.next())
+            return std::nullopt;
+        auto outputPath = parseStorePath(use.getStr(0));
+        auto resolvedDrv = StorePath(use.getStr(1));
+        return Ret{
+            Realisation{.id = id, .outPath = outputPath}};
+    });
 }
 }  // namespace nix
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 440411f01..69559e346 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -210,8 +210,8 @@ public:
 
     /* Register the store path 'output' as the output named 'outputName' of
        derivation 'deriver'. */
-    void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override;
-    void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
+    void registerDrvOutput(const Realisation & info) override;
+    void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
 
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index dfe1e2cc4..03c4351ac 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),)
 libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
 endif
 
-$(d)/local-store.cc: $(d)/schema.sql.gen.hh
+$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
 $(d)/build.cc:
 
@@ -58,7 +58,7 @@ $(d)/build.cc:
 	@echo ')foo"' >> $@.tmp
 	@mv $@.tmp $@
 
-clean-files += $(d)/schema.sql.gen.hh
+clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
 

From 8914e01e37ad072d940e2000fede7c2e0f4b194c Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 8 Dec 2020 21:07:52 +0100
Subject: [PATCH 564/882] Store the realisations as JSON in the binary cache

Fix #4332
---
 src/libstore/binary-cache-store.cc |  5 ++-
 src/libstore/realisation.cc        | 61 ++++++++++--------------------
 src/libstore/realisation.hh        |  5 ++-
 3 files changed, 25 insertions(+), 46 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 085dc7ba1..5b081c1ae 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -449,7 +449,8 @@ std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOut
     auto rawOutputInfo = getFile(outputInfoFilePath);
 
     if (rawOutputInfo) {
-        return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) };
+        return {Realisation::fromJSON(
+            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
     } else {
         return std::nullopt;
     }
@@ -457,7 +458,7 @@ std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOut
 
 void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
     auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
-    upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath");
+    upsertFile(filePath, info.toJSON(), "application/json");
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index fcc1a3825..47db1ec9f 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -1,5 +1,6 @@
 #include "realisation.hh"
 #include "store-api.hh"
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -20,52 +21,28 @@ std::string DrvOutput::to_string() const {
     return std::string(drvPath.to_string()) + "!" + outputName;
 }
 
-std::string Realisation::to_string() const {
-    std::string res;
-
-    res += "Id: " + id.to_string() + '\n';
-    res += "OutPath: " + std::string(outPath.to_string()) + '\n';
-
-    return res;
+nlohmann::json Realisation::toJSON() const {
+    return nlohmann::json{
+        {"id", id.to_string()},
+        {"outPath", outPath.to_string()},
+    };
 }
 
-Realisation Realisation::parse(const std::string & s, const std::string & whence)
-{
-    // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out
-    auto corrupt = [&]() {
-        return Error("Drv output info file '%1%' is corrupt", whence);
+Realisation Realisation::fromJSON(
+    const nlohmann::json& json,
+    const std::string& whence) {
+    auto getField = [&](std::string fieldName) -> std::string {
+        auto fieldIterator = json.find(fieldName);
+        if (fieldIterator == json.end())
+            throw Error(
+                "Drv output info file '%1%' is corrupt, missing field %2%",
+                whence, fieldName);
+        return *fieldIterator;
     };
 
-    std::optional<DrvOutput> id;
-    std::optional<StorePath> outPath;
-
-    size_t pos = 0;
-    while (pos < s.size()) {
-
-        size_t colon = s.find(':', pos);
-        if (colon == std::string::npos) throw corrupt();
-
-        std::string name(s, pos, colon - pos);
-
-        size_t eol = s.find('\n', colon + 2);
-        if (eol == std::string::npos) throw corrupt();
-
-        std::string value(s, colon + 2, eol - colon - 2);
-
-        if (name == "Id")
-            id = DrvOutput::parse(value);
-
-        if (name == "OutPath")
-            outPath = StorePath(value);
-
-        pos = eol + 1;
-    }
-
-    if (!outPath) corrupt();
-    if (!id) corrupt();
-    return Realisation {
-        .id = *id,
-        .outPath = *outPath,
+    return Realisation{
+        .id = DrvOutput::parse(getField("id")),
+        .outPath = StorePath(getField("outPath")),
     };
 }
 
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index c573e1bb4..08579b739 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "path.hh"
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 
@@ -25,8 +26,8 @@ struct Realisation {
     DrvOutput id;
     StorePath outPath;
 
-    std::string to_string() const;
-    static Realisation parse(const std::string & s, const std::string & whence);
+    nlohmann::json toJSON() const;
+    static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 };
 
 typedef std::map<DrvOutput, Realisation> DrvOutputs;

From bab1cda0e6c30e25460b5a9c809589d3948f35df Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 9 Dec 2020 16:56:56 +0100
Subject: [PATCH 565/882] Use the hash modulo in the derivation outputs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Rather than storing the derivation outputs as `drvPath!outputName` internally,
store them as `drvHashModulo!outputName` (or `outputHash!outputName` for
fixed-output derivations).

This makes the storage slightly more opaque, but enables an earlier
cutoff in cases where a fixed-output dependency changes (but keeps the
same output hash) − same as what we already do for input-addressed
derivations.
---
 src/libexpr/primops.cc                |  2 +-
 src/libstore/build/derivation-goal.cc | 28 ++++--------
 src/libstore/derivations.cc           | 49 +++++++++++++++------
 src/libstore/derivations.hh           | 22 ++++------
 src/libstore/local-store.cc           | 61 ++++++++++++---------------
 src/libstore/realisation.cc           | 10 ++---
 src/libstore/realisation.hh           | 10 +++--
 7 files changed, 93 insertions(+), 89 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 41f06c219..d059e3daf 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1107,7 +1107,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 // Shouldn't happen as the toplevel derivation is not CA.
                 assert(false);
             },
-            [&](UnknownHashes) {
+            [&](DeferredHash _) {
                 for (auto & i : outputs) {
                     drv.outputs.insert_or_assign(i,
                         DerivationOutput {
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index b7bf866eb..54b37553a 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -504,9 +504,6 @@ void DerivationGoal::inputsRealised()
             Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
-            /* Add to memotable to speed up downstream goal's queries with the
-               original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
 
             auto msg = fmt("Resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
@@ -2097,15 +2094,15 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
 
     void registerDrvOutput(const Realisation & info) override
     {
-        if (!goal.isAllowed(info.id.drvPath))
-            throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath));
+        // XXX: Should we check for something here? Probably, but I'm not sure
+        // how
         next->registerDrvOutput(info);
     }
 
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
     {
-        if (!goal.isAllowed(id.drvPath))
-            throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath));
+        // XXX: Should we check for something here? Probably, but I'm not sure
+        // how
         return next->queryRealisation(id);
     }
 
@@ -3394,23 +3391,14 @@ void DerivationGoal::registerOutputs()
        means it's safe to link the derivation to the output hash. We must do
        that for floating CA derivations, which otherwise couldn't be cached,
        but it's fine to do in all cases. */
-    bool isCaFloating = drv->type() == DerivationType::CAFloating;
 
-    auto drvPathResolved = drvPath;
-    if (!useDerivation && isCaFloating) {
-        /* Once a floating CA derivations reaches this point, it
-           must already be resolved, so we don't bother trying to
-           downcast drv to get would would just be an empty
-           inputDrvs field. */
-        Derivation drv2 { *drv };
-        drvPathResolved = writeDerivation(worker.store, drv2);
-    }
-
-    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        auto outputHashes = staticOutputHashes(worker.store, *drv);
         for (auto& [outputName, newInfo] : infos)
             worker.store.registerDrvOutput(Realisation{
-                .id = DrvOutput{drvPathResolved, outputName},
+                .id = DrvOutput{outputHashes.at(outputName), outputName},
                 .outPath = newInfo.path});
+    }
 }
 
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 231ca26c2..5bcc7f012 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -496,10 +496,9 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
  */
 DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
 {
+    bool isDeferred = false;
     /* Return a fixed hash for fixed-output derivations. */
     switch (drv.type()) {
-    case DerivationType::CAFloating:
-        return UnknownHashes {};
     case DerivationType::CAFixed: {
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
@@ -512,6 +511,9 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
         }
         return outputHashes;
     }
+    case DerivationType::CAFloating:
+        isDeferred = true;
+        break;
     case DerivationType::InputAddressed:
         break;
     case DerivationType::DeferredInputAddressed:
@@ -522,13 +524,16 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
        calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & i : drv.inputDrvs) {
-        bool hasUnknownHash = false;
         const auto & res = pathDerivationModulo(store, i.first);
         std::visit(overloaded {
             // Regular non-CA derivation, replace derivation
             [&](Hash drvHash) {
                 inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
             },
+            [&](DeferredHash deferredHash) {
+                isDeferred = true;
+                inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
+            },
             // CA derivation's output hashes
             [&](CaOutputHashes outputHashes) {
                 std::set<std::string> justOut = { "out" };
@@ -540,16 +545,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
                         justOut);
                 }
             },
-            [&](UnknownHashes) {
-                hasUnknownHash = true;
-            },
         }, res);
-        if (hasUnknownHash) {
-            return UnknownHashes {};
-        }
     }
 
-    return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
+    auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
+
+    if (isDeferred)
+        return DeferredHash { hash };
+    else
+        return hash;
+}
+
+
+std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
+{
+    std::map<std::string, Hash> res;
+    std::visit(overloaded {
+        [&](Hash drvHash) {
+            for (auto & outputName : drv.outputNames()) {
+                res.insert({outputName, drvHash});
+            }
+        },
+        [&](DeferredHash deferredHash) {
+            for (auto & outputName : drv.outputNames()) {
+                res.insert({outputName, deferredHash.hash});
+            }
+        },
+        [&](CaOutputHashes outputHashes) {
+            res = outputHashes;
+        },
+    }, hashDerivationModulo(store, drv, true));
+    return res;
 }
 
 
@@ -719,9 +745,6 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-
-Sync<DrvPathResolutions> drvPathResolutions;
-
 std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
     BasicDerivation resolved { *this };
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index b966d6d90..4e5985fab 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -18,8 +18,6 @@ namespace nix {
 /* The traditional non-fixed-output derivation type. */
 struct DerivationOutputInputAddressed
 {
-    /* Will need to become `std::optional<StorePath>` once input-addressed
-       derivations are allowed to depend on cont-addressed derivations */
     StorePath path;
 };
 
@@ -174,12 +172,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
 // whose output hashes are always known since they are fixed up-front.
 typedef std::map<std::string, Hash> CaOutputHashes;
 
-struct UnknownHashes {};
+struct DeferredHash { Hash hash; };
 
 typedef std::variant<
     Hash, // regular DRV normalized hash
     CaOutputHashes, // Fixed-output derivation hashes
-    UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies
+    DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
 > DrvHashModulo;
 
 /* Returns hashes with the details of fixed-output subderivations
@@ -207,22 +205,18 @@ typedef std::variant<
  */
 DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
 
+/*
+   Return a map associating each output to a hash that uniquely identifies its
+   derivation (modulo the self-references).
+ */
+std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
+
 /* Memoisation of hashDerivationModulo(). */
 typedef std::map<StorePath, DrvHashModulo> DrvHashes;
 
 // FIXME: global, though at least thread-safe.
 extern Sync<DrvHashes> drvHashes;
 
-/* Memoisation of `readDerivation(..).resove()`. */
-typedef std::map<
-    StorePath,
-    std::optional<StorePath>
-> DrvPathResolutions;
-
-// FIXME: global, though at least thread-safe.
-// FIXME: arguably overlaps with hashDerivationModulo memo table.
-extern Sync<DrvPathResolutions> drvPathResolutions;
-
 bool wantOutput(const string & output, const std::set<string> & wanted);
 
 struct Source;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 69ab821d9..1539c94e2 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -659,7 +659,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
     auto state(_state.lock());
     retrySQLite<void>([&]() {
         state->stmts->RegisterRealisedOutput.use()
-            (info.id.drvPath.to_string())
+            (info.id.strHash())
             (info.id.outputName)
             (printStorePath(info.outPath))
             .exec();
@@ -879,17 +879,18 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 
 // Try to resolve the derivation at path `original`, with a caching layer
 // to make it more efficient
-std::optional<StorePath> cachedResolve(
-    LocalStore & store,
-    const StorePath & original)
+std::optional<Derivation> cachedResolve(
+    LocalStore& store,
+    const StorePath& original)
 {
+    // This is quite dirty and leaky, but will disappear once #4340 is merged
+    static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
     {
-        auto resolutions = drvPathResolutions.lock();
-        auto resolvedPathOptIter = resolutions->find(original);
-        if (resolvedPathOptIter != resolutions->end()) {
-            auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
-            if (resolvedPathOpt)
-                return resolvedPathOpt;
+        auto resolutions = resolutionsCache.lock();
+        auto resolvedDrvIter = resolutions->find(original);
+        if (resolvedDrvIter != resolutions->end()) {
+            auto & [_, resolvedDrv] = *resolvedDrvIter;
+                return *resolvedDrv;
         }
     }
 
@@ -898,12 +899,9 @@ std::optional<StorePath> cachedResolve(
     auto attempt = drv.tryResolve(store);
     if (!attempt)
         return std::nullopt;
-    /* Just compute store path */
-    auto pathResolved =
-        writeDerivation(store, *std::move(attempt), NoRepair, true);
     /* Store in memo table. */
-    drvPathResolutions.lock()->insert_or_assign(original, pathResolved);
-    return pathResolved;
+    resolutionsCache.lock()->insert_or_assign(original, *attempt);
+    return *attempt;
 }
 
 std::map<std::string, std::optional<StorePath>>
@@ -933,26 +931,24 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
 
     auto drv = readDerivation(path);
 
-    for (auto & output : drv.outputsAndOptPaths(*this)) {
-        outputs.emplace(output.first, std::nullopt);
-    }
-
     auto resolvedDrv = cachedResolve(*this, path);
 
-    if (!resolvedDrv)
+    if (!resolvedDrv) {
+        for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) {
+            if (!outputs.count(outputName))
+                outputs.emplace(outputName, std::nullopt);
+        }
         return outputs;
+    }
 
-    retrySQLite<void>([&]() {
-        auto state(_state.lock());
-        path = *resolvedDrv;
-        auto useQueryDerivationOutputs{
-            state->stmts->QueryAllRealisedOutputs.use()(path.to_string())};
-
-        while (useQueryDerivationOutputs.next())
-            outputs.insert_or_assign(
-                useQueryDerivationOutputs.getStr(0),
-                parseStorePath(useQueryDerivationOutputs.getStr(1)));
-    });
+    auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv);
+    for (auto& [outputName, hash] : resolvedDrvHashes) {
+        auto realisation = queryRealisation(DrvOutput{hash, outputName});
+        if (realisation)
+            outputs.insert_or_assign(outputName, realisation->outPath);
+        else
+            outputs.insert_or_assign(outputName, std::nullopt);
+    }
 
     return outputs;
 }
@@ -1695,12 +1691,11 @@ std::optional<const Realisation> LocalStore::queryRealisation(
     typedef std::optional<const Realisation> Ret;
     return retrySQLite<Ret>([&]() -> Ret {
         auto state(_state.lock());
-        auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(
+        auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
             id.outputName));
         if (!use.next())
             return std::nullopt;
         auto outputPath = parseStorePath(use.getStr(0));
-        auto resolvedDrv = StorePath(use.getStr(1));
         return Ret{
             Realisation{.id = id, .outPath = outputPath}};
     });
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 47db1ec9f..47ad90eee 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -7,18 +7,18 @@ namespace nix {
 MakeError(InvalidDerivationOutputId, Error);
 
 DrvOutput DrvOutput::parse(const std::string &strRep) {
-    const auto &[rawPath, outputs] = parsePathWithOutputs(strRep);
-    if (outputs.size() != 1)
+    size_t n = strRep.find("!");
+    if (n == strRep.npos)
         throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
 
     return DrvOutput{
-        .drvPath = StorePath(rawPath),
-        .outputName = *outputs.begin(),
+        .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
+        .outputName = strRep.substr(n+1),
     };
 }
 
 std::string DrvOutput::to_string() const {
-    return std::string(drvPath.to_string()) + "!" + outputName;
+    return strHash() + "!" + outputName;
 }
 
 nlohmann::json Realisation::toJSON() const {
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 08579b739..4b8ead3c5 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -6,11 +6,15 @@
 namespace nix {
 
 struct DrvOutput {
-    StorePath drvPath;
+    // The hash modulo of the derivation
+    Hash drvHash;
     std::string outputName;
 
     std::string to_string() const;
 
+    std::string strHash() const
+    { return drvHash.to_string(Base16, true); }
+
     static DrvOutput parse(const std::string &);
 
     bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
@@ -18,8 +22,8 @@ struct DrvOutput {
 
 private:
     // Just to make comparison operators easier to write
-    std::pair<StorePath, std::string> to_pair() const
-    { return std::make_pair(drvPath, outputName); }
+    std::pair<Hash, std::string> to_pair() const
+    { return std::make_pair(drvHash, outputName); }
 };
 
 struct Realisation {

From e9b39f6004ec68f062230514534b08033cf133c7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Dec 2020 21:12:53 +0100
Subject: [PATCH 566/882] Restrict the operations on drv outputs in recursive
 Nix

There's currently no way to properly filter them, so disallow them
altogether instead.
---
 src/libstore/build/derivation-goal.cc | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 54b37553a..f494545fb 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -2093,18 +2093,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
     }
 
     void registerDrvOutput(const Realisation & info) override
-    {
-        // XXX: Should we check for something here? Probably, but I'm not sure
-        // how
-        next->registerDrvOutput(info);
-    }
+    // XXX: This should probably be allowed as a no-op if the realisation
+    // corresponds to an allowed derivation
+    { throw Error("registerDrvOutput"); }
 
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
-    {
-        // XXX: Should we check for something here? Probably, but I'm not sure
-        // how
-        return next->queryRealisation(id);
-    }
+    // XXX: This should probably be allowed if the realisation corresponds to
+    // an allowed derivation
+    { throw Error("queryRealisation"); }
 
     void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
     {

From fa307875e961a616a049206645a651a76a050a79 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Fri, 11 Dec 2020 23:32:45 +0100
Subject: [PATCH 567/882] Introduce NormalType for the normal type of a Value

This will be useful to abstract over the ValueType implementation
details

Make use of it already to replace the showType(ValueType) function
---
 src/libexpr/eval-cache.cc  |  4 ++--
 src/libexpr/eval.cc        | 34 ++++++++++++++++------------------
 src/libexpr/eval.hh        |  2 +-
 src/libexpr/flake/flake.cc | 26 +++++++++++++-------------
 src/libexpr/value.hh       | 36 ++++++++++++++++++++++++++++++++++++
 5 files changed, 68 insertions(+), 34 deletions(-)

diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 7b025be23..a11327f77 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -513,7 +513,7 @@ std::string AttrCursor::getString()
     auto & v = forceValue();
 
     if (v.type != tString && v.type != tPath)
-        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType()));
 
     return v.type == tString ? v.string.s : v.path;
 }
@@ -548,7 +548,7 @@ string_t AttrCursor::getStringWithContext()
     else if (v.type == tPath)
         return {v.path, {}};
     else
-        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType()));
 }
 
 bool AttrCursor::getBool()
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index c6f4d1716..48fe0bbda 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -165,25 +165,20 @@ const Value *getPrimOp(const Value &v) {
     return primOp;
 }
 
-
-string showType(ValueType type)
+string showType(NormalType type)
 {
     switch (type) {
-        case tInt: return "an integer";
-        case tBool: return "a Boolean";
-        case tString: return "a string";
-        case tPath: return "a path";
-        case tNull: return "null";
-        case tAttrs: return "a set";
-        case tList1: case tList2: case tListN: return "a list";
-        case tThunk: return "a thunk";
-        case tApp: return "a function application";
-        case tLambda: return "a function";
-        case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
-        case tExternal: return "an external value";
-        case tFloat: return "a float";
+        case nInt: return "an integer";
+        case nBool: return "a Boolean";
+        case nString: return "a string";
+        case nPath: return "a path";
+        case nNull: return "null";
+        case nAttrs: return "a set";
+        case nList: return "a list";
+        case nFunction: return "a function";
+        case nExternal: return "an external value";
+        case nFloat: return "a float";
+        case nThunk: return "a thunk";
     }
     abort();
 }
@@ -198,8 +193,11 @@ string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
+        case tThunk: return "a thunk";
+        case tApp: return "a function application";
+        case tBlackhole: return "a black hole";
     default:
-        return showType(v.type);
+        return showType(v.normalType());
     }
 }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 0e1f61baa..211529954 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -346,7 +346,7 @@ private:
 
 
 /* Return a string representing the type of the value `v'. */
-string showType(ValueType type);
+string showType(NormalType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 3e866e1f9..881b1b4e5 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -78,13 +78,13 @@ static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
 }
 
 
-static void expectType(EvalState & state, ValueType type,
+static void expectType(EvalState & state, NormalType type,
     Value & value, const Pos & pos)
 {
     forceTrivialValue(state, value, pos);
-    if (value.type != type)
+    if (value.normalType() != type)
         throw Error("expected %s but got %s at %s",
-            showType(type), showType(value.type), pos);
+            showType(type), showType(value.normalType()), pos);
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
@@ -93,7 +93,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 static FlakeInput parseFlakeInput(EvalState & state,
     const std::string & inputName, Value * value, const Pos & pos)
 {
-    expectType(state, tAttrs, *value, pos);
+    expectType(state, nAttrs, *value, pos);
 
     FlakeInput input;
 
@@ -108,16 +108,16 @@ static FlakeInput parseFlakeInput(EvalState & state,
     for (nix::Attr attr : *(value->attrs)) {
         try {
             if (attr.name == sUrl) {
-                expectType(state, tString, *attr.value, *attr.pos);
+                expectType(state, nString, *attr.value, *attr.pos);
                 url = attr.value->string.s;
                 attrs.emplace("url", *url);
             } else if (attr.name == sFlake) {
-                expectType(state, tBool, *attr.value, *attr.pos);
+                expectType(state, nBool, *attr.value, *attr.pos);
                 input.isFlake = attr.value->boolean;
             } else if (attr.name == sInputs) {
                 input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
             } else if (attr.name == sFollows) {
-                expectType(state, tString, *attr.value, *attr.pos);
+                expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
                 if (attr.value->type == tString)
@@ -158,7 +158,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 {
     std::map<FlakeId, FlakeInput> inputs;
 
-    expectType(state, tAttrs, *value, pos);
+    expectType(state, nAttrs, *value, pos);
 
     for (nix::Attr & inputAttr : *(*value).attrs) {
         inputs.emplace(inputAttr.name,
@@ -199,10 +199,10 @@ static Flake getFlake(
     Value vInfo;
     state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
 
-    expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
+    expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
 
     if (auto description = vInfo.attrs->get(state.sDescription)) {
-        expectType(state, tString, *description->value, *description->pos);
+        expectType(state, nString, *description->value, *description->pos);
         flake.description = description->value->string.s;
     }
 
@@ -214,9 +214,9 @@ static Flake getFlake(
     auto sOutputs = state.symbols.create("outputs");
 
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
-        expectType(state, tLambda, *outputs->value, *outputs->pos);
+        expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->lambda.fun->matchAttrs) {
+        if (outputs->value->type == tLambda && outputs->value->lambda.fun->matchAttrs) {
             for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
@@ -231,7 +231,7 @@ static Flake getFlake(
     auto sNixConfig = state.symbols.create("nixConfig");
 
     if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
-        expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos);
+        expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos);
 
         for (auto & setting : *nixConfig->value->attrs) {
             forceTrivialValue(state, *setting.value, *setting.pos);
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index fe11bb2ed..833af0f3d 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -29,6 +29,22 @@ typedef enum {
     tFloat
 } ValueType;
 
+// This type abstracts over all actual value types in the language,
+// grouping together implementation details like tList*, different function
+// types, and types in non-normal form (so thunks and co.)
+typedef enum {
+    nThunk,
+    nInt,
+    nFloat,
+    nBool,
+    nString,
+    nPath,
+    nNull,
+    nAttrs,
+    nList,
+    nFunction,
+    nExternal
+} NormalType;
 
 class Bindings;
 struct Env;
@@ -147,6 +163,26 @@ struct Value
         NixFloat fpoint;
     };
 
+    // Returns the normal type of a Value. This only returns nThunk if the
+    // Value hasn't been forceValue'd
+    inline NormalType normalType() const
+    {
+        switch (type) {
+            case tInt: return nInt;
+            case tBool: return nBool;
+            case tString: return nString;
+            case tPath: return nPath;
+            case tNull: return nNull;
+            case tAttrs: return nAttrs;
+            case tList1: case tList2: case tListN: return nList;
+            case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
+            case tExternal: return nExternal;
+            case tFloat: return nFloat;
+            case tThunk: case tApp: case tBlackhole: return nThunk;
+        }
+        abort();
+    }
+
     bool isList() const
     {
         return type == tList1 || type == tList2 || type == tListN;

From 9f056f7afdb85b8c3bd59638197e356f269129b2 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Sat, 12 Dec 2020 00:19:05 +0100
Subject: [PATCH 568/882] Introduce Value type setters and make use of them

---
 src/libexpr/attr-set.cc    |  2 +-
 src/libexpr/eval-inline.hh |  4 ++--
 src/libexpr/eval.cc        | 24 ++++++++++++------------
 src/libexpr/value.hh       | 35 +++++++++++++++++++++++++++--------
 src/nix/repl.cc            |  2 +-
 5 files changed, 43 insertions(+), 24 deletions(-)

diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index b1d61a285..17886a426 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -25,7 +25,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
         return;
     }
     clearValue(v);
-    v.type = tAttrs;
+    v.setAttrs();
     v.attrs = allocBindings(capacity);
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 30f6ec7db..a0fd9b569 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -36,11 +36,11 @@ void EvalState::forceValue(Value & v, const Pos & pos)
         Env * env = v.thunk.env;
         Expr * expr = v.thunk.expr;
         try {
-            v.type = tBlackhole;
+            v.setBlackhole();
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
-            v.type = tThunk;
+            v.setThunk();
             v.thunk.env = env;
             v.thunk.expr = expr;
             throw;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 48fe0bbda..f68828944 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -431,7 +431,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     }
 
     clearValue(vEmptySet);
-    vEmptySet.type = tAttrs;
+    vEmptySet.setAttrs();
     vEmptySet.attrs = allocBindings(0);
 
     createBaseEnv();
@@ -548,7 +548,7 @@ Value * EvalState::addPrimOp(const string & name,
        the primop to a dummy value. */
     if (arity == 0) {
         auto vPrimOp = allocValue();
-        vPrimOp->type = tPrimOp;
+        vPrimOp->setPrimOp();
         vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
@@ -556,7 +556,7 @@ Value * EvalState::addPrimOp(const string & name,
     }
 
     Value * v = allocValue();
-    v->type = tPrimOp;
+    v->setPrimOp();
     v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
@@ -572,7 +572,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     if (primOp.arity == 0) {
         primOp.arity = 1;
         auto vPrimOp = allocValue();
-        vPrimOp->type = tPrimOp;
+        vPrimOp->setPrimOp();
         vPrimOp->primOp = new PrimOp(std::move(primOp));
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
@@ -584,7 +584,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         primOp.name = symbols.create(std::string(primOp.name, 2));
 
     Value * v = allocValue();
-    v->type = tPrimOp;
+    v->setPrimOp();
     v->primOp = new PrimOp(std::move(primOp));
     staticBaseEnv.vars[envName] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
@@ -714,7 +714,7 @@ void mkString(Value & v, const char * s)
 
 Value & mkString(Value & v, std::string_view s, const PathSet & context)
 {
-    v.type = tString;
+    v.setString();
     v.string.s = dupStringWithLen(s.data(), s.size());
     v.string.context = 0;
     if (!context.empty()) {
@@ -794,11 +794,11 @@ void EvalState::mkList(Value & v, size_t size)
 {
     clearValue(v);
     if (size == 1)
-        v.type = tList1;
+        v.setList1();
     else if (size == 2)
-        v.type = tList2;
+        v.setList2();
     else {
-        v.type = tListN;
+        v.setListN();
         v.bigList.size = size;
         v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0;
     }
@@ -810,7 +810,7 @@ unsigned long nrThunks = 0;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
-    v.type = tThunk;
+    v.setThunk();
     v.thunk.env = &env;
     v.thunk.expr = expr;
     nrThunks++;
@@ -1207,7 +1207,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
 
 void ExprLambda::eval(EvalState & state, Env & env, Value & v)
 {
-    v.type = tLambda;
+    v.setLambda();
     v.lambda.env = &env;
     v.lambda.fun = this;
 }
@@ -1252,7 +1252,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     } else {
         Value * fun2 = allocValue();
         *fun2 = fun;
-        v.type = tPrimOpApp;
+        v.setPrimOpApp();
         v.primOpApp.left = fun2;
         v.primOpApp.right = &arg;
     }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 833af0f3d..0995dcd7b 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -107,6 +107,25 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 struct Value
 {
     ValueType type;
+
+    inline void setInt() { type = tInt; };
+    inline void setBool() { type = tBool; };
+    inline void setString() { type = tString; };
+    inline void setPath() { type = tPath; };
+    inline void setNull() { type = tNull; };
+    inline void setAttrs() { type = tAttrs; };
+    inline void setList1() { type = tList1; };
+    inline void setList2() { type = tList2; };
+    inline void setListN() { type = tListN; };
+    inline void setThunk() { type = tThunk; };
+    inline void setApp() { type = tApp; };
+    inline void setLambda() { type = tLambda; };
+    inline void setBlackhole() { type = tBlackhole; };
+    inline void setPrimOp() { type = tPrimOp; };
+    inline void setPrimOpApp() { type = tPrimOpApp; };
+    inline void setExternal() { type = tExternal; };
+    inline void setFloat() { type = tFloat; };
+
     union
     {
         NixInt integer;
@@ -223,7 +242,7 @@ static inline void clearValue(Value & v)
 static inline void mkInt(Value & v, NixInt n)
 {
     clearValue(v);
-    v.type = tInt;
+    v.setInt();
     v.integer = n;
 }
 
@@ -231,7 +250,7 @@ static inline void mkInt(Value & v, NixInt n)
 static inline void mkFloat(Value & v, NixFloat n)
 {
     clearValue(v);
-    v.type = tFloat;
+    v.setFloat();
     v.fpoint = n;
 }
 
@@ -239,7 +258,7 @@ static inline void mkFloat(Value & v, NixFloat n)
 static inline void mkBool(Value & v, bool b)
 {
     clearValue(v);
-    v.type = tBool;
+    v.setBool();
     v.boolean = b;
 }
 
@@ -247,13 +266,13 @@ static inline void mkBool(Value & v, bool b)
 static inline void mkNull(Value & v)
 {
     clearValue(v);
-    v.type = tNull;
+    v.setNull();
 }
 
 
 static inline void mkApp(Value & v, Value & left, Value & right)
 {
-    v.type = tApp;
+    v.setApp();
     v.app.left = &left;
     v.app.right = &right;
 }
@@ -261,7 +280,7 @@ static inline void mkApp(Value & v, Value & left, Value & right)
 
 static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
 {
-    v.type = tPrimOpApp;
+    v.setPrimOpApp();
     v.app.left = &left;
     v.app.right = &right;
 }
@@ -269,7 +288,7 @@ static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
 
 static inline void mkStringNoCopy(Value & v, const char * s)
 {
-    v.type = tString;
+    v.setString();
     v.string.s = s;
     v.string.context = 0;
 }
@@ -287,7 +306,7 @@ void mkString(Value & v, const char * s);
 static inline void mkPathNoCopy(Value & v, const char * s)
 {
     clearValue(v);
-    v.type = tPath;
+    v.setPath();
     v.path = s;
 }
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 71794a309..3cee81b49 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -551,7 +551,7 @@ bool NixRepl::processLine(string line)
         {
             Expr * e = parseString(string(line, p + 1));
             Value & v(*state->allocValue());
-            v.type = tThunk;
+            v.setThunk();
             v.thunk.env = env;
             v.thunk.expr = e;
             addVarToScope(state->symbols.create(name), v);

From 22ead43a0b8f94f5a4fb64cff14bf6a2a35d671c Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Sat, 12 Dec 2020 02:09:10 +0100
Subject: [PATCH 569/882] Use Value::normalType on all forced values instead of
 Value::type

---
 src/libexpr/attr-path.cc              |   2 +-
 src/libexpr/eval-cache.cc             |  22 ++---
 src/libexpr/eval-inline.hh            |   4 +-
 src/libexpr/eval.cc                   | 112 +++++++++++++-------------
 src/libexpr/flake/flake.cc            |  12 +--
 src/libexpr/get-drvs.cc               |  30 +++----
 src/libexpr/primops.cc                |  89 +++++++++-----------
 src/libexpr/primops/fetchMercurial.cc |   2 +-
 src/libexpr/primops/fetchTree.cc      |  12 +--
 src/libexpr/value-to-json.cc          |  25 +++---
 src/libexpr/value-to-xml.cc           |  33 ++++----
 src/nix-env/nix-env.cc                |  16 ++--
 src/nix/eval.cc                       |   4 +-
 src/nix/flake.cc                      |   2 +-
 src/nix/repl.cc                       |  47 +++++------
 15 files changed, 199 insertions(+), 213 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 83854df49..54e13e6a2 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -67,7 +67,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
         if (apType == apAttr) {
 
-            if (v->type != tAttrs)
+            if (v->normalType() != nAttrs)
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a set but is %2%",
                     attrPath,
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index a11327f77..3c97f1201 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -390,14 +390,14 @@ Value & AttrCursor::forceValue()
     }
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
-        if (v.type == tString)
+        if (v.normalType() == nString)
             cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
                            string_t{v.string.s, {}}};
-        else if (v.type == tPath)
+        else if (v.normalType() == nPath)
             cachedValue = {root->db->setString(getKey(), v.path), v.path};
-        else if (v.type == tBool)
+        else if (v.normalType() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
-        else if (v.type == tAttrs)
+        else if (v.normalType() == nAttrs)
             ; // FIXME: do something?
         else
             cachedValue = {root->db->setMisc(getKey()), misc_t()};
@@ -442,7 +442,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
 
     auto & v = forceValue();
 
-    if (v.type != tAttrs)
+    if (v.normalType() != nAttrs)
         return nullptr;
         //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
@@ -512,10 +512,10 @@ std::string AttrCursor::getString()
 
     auto & v = forceValue();
 
-    if (v.type != tString && v.type != tPath)
+    if (v.normalType() != nString && v.normalType() != nPath)
         throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType()));
 
-    return v.type == tString ? v.string.s : v.path;
+    return v.normalType() == nString ? v.string.s : v.path;
 }
 
 string_t AttrCursor::getStringWithContext()
@@ -543,9 +543,9 @@ string_t AttrCursor::getStringWithContext()
 
     auto & v = forceValue();
 
-    if (v.type == tString)
+    if (v.normalType() == nString)
         return {v.string.s, v.getContext()};
-    else if (v.type == tPath)
+    else if (v.normalType() == nPath)
         return {v.path, {}};
     else
         throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType()));
@@ -567,7 +567,7 @@ bool AttrCursor::getBool()
 
     auto & v = forceValue();
 
-    if (v.type != tBool)
+    if (v.normalType() != nBool)
         throw TypeError("'%s' is not a Boolean", getAttrPathStr());
 
     return v.boolean;
@@ -589,7 +589,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
 
     auto & v = forceValue();
 
-    if (v.type != tAttrs)
+    if (v.normalType() != nAttrs)
         throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
     std::vector<Symbol> attrs;
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index a0fd9b569..9b644d5cb 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -56,7 +56,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
 inline void EvalState::forceAttrs(Value & v)
 {
     forceValue(v);
-    if (v.type != tAttrs)
+    if (v.normalType() != nAttrs)
         throwTypeError("value is %1% while a set was expected", v);
 }
 
@@ -64,7 +64,7 @@ inline void EvalState::forceAttrs(Value & v)
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tAttrs)
+    if (v.normalType() != nAttrs)
         throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f68828944..f33426b59 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -947,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e)
 {
     Value v;
     e->eval(*this, env, v);
-    if (v.type != tBool)
+    if (v.normalType() != nBool)
         throwTypeError("value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -957,7 +957,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
 {
     Value v;
     e->eval(*this, env, v);
-    if (v.type != tBool)
+    if (v.normalType() != nBool)
         throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -966,7 +966,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
 inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v)
 {
     e->eval(*this, env, v);
-    if (v.type != tAttrs)
+    if (v.normalType() != nAttrs)
         throwTypeError("value is %1% while a set was expected", v);
 }
 
@@ -1066,7 +1066,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Value nameVal;
         i.nameExpr->eval(state, *dynamicEnv, nameVal);
         state.forceValue(nameVal, i.pos);
-        if (nameVal.type == tNull)
+        if (nameVal.normalType() == nNull)
             continue;
         state.forceStringNoCtx(nameVal);
         Symbol nameSym = state.symbols.create(nameVal.string.s);
@@ -1151,7 +1151,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             Symbol name = getName(i, state, env);
             if (def) {
                 state.forceValue(*vAttrs, pos);
-                if (vAttrs->type != tAttrs ||
+                if (vAttrs->normalType() != nAttrs ||
                     (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
                 {
                     def->eval(state, env, v);
@@ -1191,7 +1191,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
         state.forceValue(*vAttrs);
         Bindings::iterator j;
         Symbol name = getName(i, state, env);
-        if (vAttrs->type != tAttrs ||
+        if (vAttrs->normalType() != nAttrs ||
             (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
         {
             mkBool(v, false);
@@ -1269,7 +1269,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         return;
     }
 
-    if (fun.type == tAttrs) {
+    if (fun.normalType() == nAttrs) {
       auto found = fun.attrs->find(sFunctor);
       if (found != fun.attrs->end()) {
         /* fun may be allocated on the stack of the calling function,
@@ -1368,7 +1368,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
     forceValue(fun);
 
-    if (fun.type == tAttrs) {
+    if (fun.normalType() == nAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
             Value * v = allocValue();
@@ -1562,7 +1562,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     NixFloat nf = 0;
 
     bool first = !forceString;
-    ValueType firstType = tString;
+    NormalType firstType = nString;
 
     for (auto & i : *es) {
         Value vTmp;
@@ -1573,36 +1573,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
-            firstType = vTmp.type;
+            firstType = vTmp.normalType();
             first = false;
         }
 
-        if (firstType == tInt) {
-            if (vTmp.type == tInt) {
+        if (firstType == nInt) {
+            if (vTmp.normalType() == nInt) {
                 n += vTmp.integer;
-            } else if (vTmp.type == tFloat) {
+            } else if (vTmp.normalType() == nFloat) {
                 // Upgrade the type from int to float;
-                firstType = tFloat;
+                firstType = nFloat;
                 nf = n;
                 nf += vTmp.fpoint;
             } else
                 throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
-        } else if (firstType == tFloat) {
-            if (vTmp.type == tInt) {
+        } else if (firstType == nFloat) {
+            if (vTmp.normalType() == nInt) {
                 nf += vTmp.integer;
-            } else if (vTmp.type == tFloat) {
+            } else if (vTmp.normalType() == nFloat) {
                 nf += vTmp.fpoint;
             } else
                 throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
-            s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
+            s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
     }
 
-    if (firstType == tInt)
+    if (firstType == nInt)
         mkInt(v, n);
-    else if (firstType == tFloat)
+    else if (firstType == nFloat)
         mkFloat(v, nf);
-    else if (firstType == tPath) {
+    else if (firstType == nPath) {
         if (!context.empty())
             throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
         auto path = canonPath(s.str());
@@ -1629,7 +1629,7 @@ void EvalState::forceValueDeep(Value & v)
 
         forceValue(v);
 
-        if (v.type == tAttrs) {
+        if (v.normalType() == nAttrs) {
             for (auto & i : *v.attrs)
                 try {
                     recurse(*i.value);
@@ -1652,7 +1652,7 @@ void EvalState::forceValueDeep(Value & v)
 NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tInt)
+    if (v.normalType() != nInt)
         throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
@@ -1661,9 +1661,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type == tInt)
+    if (v.normalType() == nInt)
         return v.integer;
-    else if (v.type != tFloat)
+    else if (v.normalType() != nFloat)
         throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
@@ -1672,7 +1672,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tBool)
+    if (v.normalType() != nBool)
         throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -1680,14 +1680,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 
 bool EvalState::isFunctor(Value & fun)
 {
-    return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
+    return fun.normalType() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
 }
 
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
+    if (v.normalType() != nFunction && !isFunctor(v))
         throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
@@ -1695,7 +1695,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 string EvalState::forceString(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tString) {
+    if (v.normalType() != nString) {
         if (pos)
             throwTypeError(pos, "value is %1% while a string was expected", v);
         else
@@ -1761,11 +1761,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
 
 bool EvalState::isDerivation(Value & v)
 {
-    if (v.type != tAttrs) return false;
+    if (v.normalType() != nAttrs) return false;
     Bindings::iterator i = v.attrs->find(sType);
     if (i == v.attrs->end()) return false;
     forceValue(*i->value);
-    if (i->value->type != tString) return false;
+    if (i->value->normalType() != nString) return false;
     return strcmp(i->value->string.s, "derivation") == 0;
 }
 
@@ -1790,17 +1790,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
     string s;
 
-    if (v.type == tString) {
+    if (v.normalType() == nString) {
         copyContext(v, context);
         return v.string.s;
     }
 
-    if (v.type == tPath) {
+    if (v.normalType() == nPath) {
         Path path(canonPath(v.path));
         return copyToStore ? copyPathToStore(context, path) : path;
     }
 
-    if (v.type == tAttrs) {
+    if (v.normalType() == nAttrs) {
         auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
         if (maybeString) {
             return *maybeString;
@@ -1810,18 +1810,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
-    if (v.type == tExternal)
+    if (v.normalType() == nExternal)
         return v.external->coerceToString(pos, context, coerceMore, copyToStore);
 
     if (coerceMore) {
 
         /* Note that `false' is represented as an empty string for
            shell scripting convenience, just like `null'. */
-        if (v.type == tBool && v.boolean) return "1";
-        if (v.type == tBool && !v.boolean) return "";
-        if (v.type == tInt) return std::to_string(v.integer);
-        if (v.type == tFloat) return std::to_string(v.fpoint);
-        if (v.type == tNull) return "";
+        if (v.normalType() == nBool && v.boolean) return "1";
+        if (v.normalType() == nBool && !v.boolean) return "";
+        if (v.normalType() == nInt) return std::to_string(v.integer);
+        if (v.normalType() == nFloat) return std::to_string(v.fpoint);
+        if (v.normalType() == nNull) return "";
 
         if (v.isList()) {
             string result;
@@ -1884,40 +1884,38 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     if (&v1 == &v2) return true;
 
     // Special case type-compatibility between float and int
-    if (v1.type == tInt && v2.type == tFloat)
+    if (v1.normalType() == nInt && v2.normalType() == nFloat)
         return v1.integer == v2.fpoint;
-    if (v1.type == tFloat && v2.type == tInt)
+    if (v1.normalType() == nFloat && v2.normalType() == nInt)
         return v1.fpoint == v2.integer;
 
     // All other types are not compatible with each other.
-    if (v1.type != v2.type) return false;
+    if (v1.normalType() != v2.normalType()) return false;
 
-    switch (v1.type) {
+    switch (v1.normalType()) {
 
-        case tInt:
+        case nInt:
             return v1.integer == v2.integer;
 
-        case tBool:
+        case nBool:
             return v1.boolean == v2.boolean;
 
-        case tString:
+        case nString:
             return strcmp(v1.string.s, v2.string.s) == 0;
 
-        case tPath:
+        case nPath:
             return strcmp(v1.path, v2.path) == 0;
 
-        case tNull:
+        case nNull:
             return true;
 
-        case tList1:
-        case tList2:
-        case tListN:
+        case nList:
             if (v1.listSize() != v2.listSize()) return false;
             for (size_t n = 0; n < v1.listSize(); ++n)
                 if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
             return true;
 
-        case tAttrs: {
+        case nAttrs: {
             /* If both sets denote a derivation (type = "derivation"),
                then compare their outPaths. */
             if (isDerivation(v1) && isDerivation(v2)) {
@@ -1939,15 +1937,13 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         }
 
         /* Functions are incomparable. */
-        case tLambda:
-        case tPrimOp:
-        case tPrimOpApp:
+        case nFunction:
             return false;
 
-        case tExternal:
+        case nExternal:
             return *v1.external == *v2.external;
 
-        case tFloat:
+        case nFloat:
             return v1.fpoint == v2.fpoint;
 
         default:
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 881b1b4e5..c126b2c40 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -120,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
-                if (attr.value->type == tString)
+                if (attr.value->normalType() == nString)
                     attrs.emplace(attr.name, attr.value->string.s);
                 else
                     throw TypeError("flake input attribute '%s' is %s while a string is expected",
@@ -235,17 +235,17 @@ static Flake getFlake(
 
         for (auto & setting : *nixConfig->value->attrs) {
             forceTrivialValue(state, *setting.value, *setting.pos);
-            if (setting.value->type == tString)
+            if (setting.value->normalType() == nString)
                 flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
-            else if (setting.value->type == tInt)
+            else if (setting.value->normalType() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
-            else if (setting.value->type == tBool)
+            else if (setting.value->normalType() == nBool)
                 flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
-            else if (setting.value->isList()) {
+            else if (setting.value->normalType() == nList) {
                 std::vector<std::string> ss;
                 for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
                     auto elem = setting.value->listElems()[n];
-                    if (elem->type != tString)
+                    if (elem->normalType() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
                     ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 91916e8bf..93788273f 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->type != tString) throw errMsg;
+        if ((*i)->normalType() != nString) throw errMsg;
         auto out = outputs.find((*i)->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
@@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames()
 bool DrvInfo::checkMeta(Value & v)
 {
     state->forceValue(v);
-    if (v.isList()) {
+    if (v.normalType() == nList) {
         for (unsigned int n = 0; n < v.listSize(); ++n)
             if (!checkMeta(*v.listElems()[n])) return false;
         return true;
     }
-    else if (v.type == tAttrs) {
+    else if (v.normalType() == nAttrs) {
         Bindings::iterator i = v.attrs->find(state->sOutPath);
         if (i != v.attrs->end()) return false;
         for (auto & i : *v.attrs)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.type == tInt || v.type == tBool || v.type == tString ||
-                v.type == tFloat;
+    else return v.normalType() == nInt || v.normalType() == nBool || v.normalType() == nString ||
+                v.normalType() == nFloat;
 }
 
 
@@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name)
 string DrvInfo::queryMetaString(const string & name)
 {
     Value * v = queryMeta(name);
-    if (!v || v->type != tString) return "";
+    if (!v || v->normalType() != nString) return "";
     return v->string.s;
 }
 
@@ -210,8 +210,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type == tInt) return v->integer;
-    if (v->type == tString) {
+    if (v->normalType() == nInt) return v->integer;
+    if (v->normalType() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
         NixInt n;
@@ -224,8 +224,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type == tFloat) return v->fpoint;
-    if (v->type == tString) {
+    if (v->normalType() == nFloat) return v->fpoint;
+    if (v->normalType() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
         NixFloat n;
@@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type == tBool) return v->boolean;
-    if (v->type == tString) {
+    if (v->normalType() == nBool) return v->boolean;
+    if (v->normalType() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
         if (strcmp(v->string.s, "true") == 0) return true;
@@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn,
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v.type == tAttrs) {
+    else if (v.normalType() == nAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
@@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                 /* If the value of this attribute is itself a set,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
-                if (i->value->type == tAttrs) {
+                if (i->value->normalType() == nAttrs) {
                     Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
@@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v.isList()) {
+    else if (v.normalType() == nList) {
         for (unsigned int n = 0; n < v.listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
             if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 41f06c219..d501f7482 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -356,24 +356,20 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
 {
     state.forceValue(*args[0], pos);
     string t;
-    switch (args[0]->type) {
-        case tInt: t = "int"; break;
-        case tBool: t = "bool"; break;
-        case tString: t = "string"; break;
-        case tPath: t = "path"; break;
-        case tNull: t = "null"; break;
-        case tAttrs: t = "set"; break;
-        case tList1: case tList2: case tListN: t = "list"; break;
-        case tLambda:
-        case tPrimOp:
-        case tPrimOpApp:
-            t = "lambda";
-            break;
-        case tExternal:
+    switch (args[0]->normalType()) {
+        case nInt: t = "int"; break;
+        case nBool: t = "bool"; break;
+        case nString: t = "string"; break;
+        case nPath: t = "path"; break;
+        case nNull: t = "null"; break;
+        case nAttrs: t = "set"; break;
+        case nList: t = "list"; break;
+        case nFunction: t = "lambda"; break;
+        case nExternal:
             t = args[0]->external->typeOf();
             break;
-        case tFloat: t = "float"; break;
-        default: abort();
+        case nFloat: t = "float"; break;
+        case nThunk: abort();
     }
     mkString(v, state.symbols.create(t));
 }
@@ -393,7 +389,7 @@ static RegisterPrimOp primop_typeOf({
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tNull);
+    mkBool(v, args[0]->normalType() == nNull);
 }
 
 static RegisterPrimOp primop_isNull({
@@ -413,18 +409,7 @@ static RegisterPrimOp primop_isNull({
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    bool res;
-    switch (args[0]->type) {
-        case tLambda:
-        case tPrimOp:
-        case tPrimOpApp:
-            res = true;
-            break;
-        default:
-            res = false;
-            break;
-    }
-    mkBool(v, res);
+    mkBool(v, args[0]->normalType() == nFunction);
 }
 
 static RegisterPrimOp primop_isFunction({
@@ -440,7 +425,7 @@ static RegisterPrimOp primop_isFunction({
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tInt);
+    mkBool(v, args[0]->normalType() == nInt);
 }
 
 static RegisterPrimOp primop_isInt({
@@ -456,7 +441,7 @@ static RegisterPrimOp primop_isInt({
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tFloat);
+    mkBool(v, args[0]->normalType() == nFloat);
 }
 
 static RegisterPrimOp primop_isFloat({
@@ -472,7 +457,7 @@ static RegisterPrimOp primop_isFloat({
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tString);
+    mkBool(v, args[0]->normalType() == nString);
 }
 
 static RegisterPrimOp primop_isString({
@@ -488,7 +473,7 @@ static RegisterPrimOp primop_isString({
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tBool);
+    mkBool(v, args[0]->normalType() == nBool);
 }
 
 static RegisterPrimOp primop_isBool({
@@ -504,7 +489,7 @@ static RegisterPrimOp primop_isBool({
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tPath);
+    mkBool(v, args[0]->normalType() == nPath);
 }
 
 static RegisterPrimOp primop_isPath({
@@ -520,20 +505,20 @@ struct CompareValues
 {
     bool operator () (const Value * v1, const Value * v2) const
     {
-        if (v1->type == tFloat && v2->type == tInt)
+        if (v1->normalType() == nFloat && v2->normalType() == nInt)
             return v1->fpoint < v2->integer;
-        if (v1->type == tInt && v2->type == tFloat)
+        if (v1->normalType() == nInt && v2->normalType() == nFloat)
             return v1->integer < v2->fpoint;
-        if (v1->type != v2->type)
+        if (v1->normalType() != v2->normalType())
             throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
-        switch (v1->type) {
-            case tInt:
+        switch (v1->normalType()) {
+            case nInt:
                 return v1->integer < v2->integer;
-            case tFloat:
+            case nFloat:
                 return v1->fpoint < v2->fpoint;
-            case tString:
+            case nString:
                 return strcmp(v1->string.s, v2->string.s) < 0;
-            case tPath:
+            case nPath:
                 return strcmp(v1->path, v2->path) < 0;
             default:
                 throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
@@ -777,7 +762,7 @@ static RegisterPrimOp primop_deepSeq({
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    if (args[0]->type == tString)
+    if (args[0]->normalType() == nString)
         printError("trace: %1%", args[0]->string.s);
     else
         printError("trace: %1%", *args[0]);
@@ -902,7 +887,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
             if (ignoreNulls) {
                 state.forceValue(*i->value, pos);
-                if (i->value->type == tNull) continue;
+                if (i->value->normalType() == nNull) continue;
             }
 
             if (i->name == state.sContentAddressed) {
@@ -1308,7 +1293,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
-    if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
+    if (args[0]->normalType() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
 static RegisterPrimOp primop_dirOf({
@@ -1808,7 +1793,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         });
 
     state.forceValue(*args[0], pos);
-    if (args[0]->type != tLambda)
+    if (args[0]->normalType() != nFunction)
         throw TypeError({
             .hint = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
@@ -2074,7 +2059,7 @@ static RegisterPrimOp primop_hasAttr({
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tAttrs);
+    mkBool(v, args[0]->normalType() == nAttrs);
 }
 
 static RegisterPrimOp primop_isAttrs({
@@ -2337,7 +2322,7 @@ static RegisterPrimOp primop_mapAttrs({
 static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->isList());
+    mkBool(v, args[0]->normalType() == nList);
 }
 
 static RegisterPrimOp primop_isList({
@@ -2831,7 +2816,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->type == tFloat || args[1]->type == tFloat)
+    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos));
@@ -2850,7 +2835,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->type == tFloat || args[1]->type == tFloat)
+    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos));
@@ -2869,7 +2854,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->type == tFloat || args[1]->type == tFloat)
+    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos));
@@ -2896,7 +2881,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
             .errPos = pos
         });
 
-    if (args[0]->type == tFloat || args[1]->type == tFloat) {
+    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
     } else {
         NixInt i1 = state.forceInt(*args[0], pos);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a77035c16..2461ebc99 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
 
     state.forceValue(*args[0]);
 
-    if (args[0]->type == tAttrs) {
+    if (args[0]->normalType() == nAttrs) {
 
         state.forceAttrs(*args[0], pos);
 
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index d094edf92..6d93e1dc2 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -85,25 +85,25 @@ static void fetchTree(
 
     state.forceValue(*args[0]);
 
-    if (args[0]->type == tAttrs) {
+    if (args[0]->normalType() == nAttrs) {
         state.forceAttrs(*args[0], pos);
 
         fetchers::Attrs attrs;
 
         for (auto & attr : *args[0]->attrs) {
             state.forceValue(*attr.value);
-            if (attr.value->type == tPath || attr.value->type == tString)
+            if (attr.value->normalType() == nPath || attr.value->normalType() == nString)
                 addURI(
                     state,
                     attrs,
                     attr.name,
                     state.coerceToString(*attr.pos, *attr.value, context, false, false)
                 );
-            else if (attr.value->type == tString)
+            else if (attr.value->normalType() == nString)
                 addURI(state, attrs, attr.name, attr.value->string.s);
-            else if (attr.value->type == tBool)
+            else if (attr.value->normalType() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
-            else if (attr.value->type == tInt)
+            else if (attr.value->normalType() == nInt)
                 attrs.emplace(attr.name, attr.value->integer);
             else
                 throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
@@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.forceValue(*args[0]);
 
-    if (args[0]->type == tAttrs) {
+    if (args[0]->normalType() == nAttrs) {
 
         state.forceAttrs(*args[0], pos);
 
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index 6ec8315ba..b5f4c8654 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict,
 
     if (strict) state.forceValue(v);
 
-    switch (v.type) {
+    switch (v.normalType()) {
 
-        case tInt:
+        case nInt:
             out.write(v.integer);
             break;
 
-        case tBool:
+        case nBool:
             out.write(v.boolean);
             break;
 
-        case tString:
+        case nString:
             copyContext(v, context);
             out.write(v.string.s);
             break;
 
-        case tPath:
+        case nPath:
             out.write(state.copyPathToStore(context, v.path));
             break;
 
-        case tNull:
+        case nNull:
             out.write(nullptr);
             break;
 
-        case tAttrs: {
+        case nAttrs: {
             auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
             if (maybeString) {
                 out.write(*maybeString);
@@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tList1: case tList2: case tListN: {
+        case nList: {
             auto list(out.list());
             for (unsigned int n = 0; n < v.listSize(); ++n) {
                 auto placeholder(list.placeholder());
@@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tExternal:
+        case nExternal:
             v.external->printValueAsJSON(state, strict, out, context);
             break;
 
-        case tFloat:
+        case nFloat:
             out.write(v.fpoint);
             break;
 
-        default:
+        case nThunk:
+            throw TypeError("cannot convert %1% to JSON", showType(v));
+
+        case nFunction:
             throw TypeError("cannot convert %1% to JSON", showType(v));
     }
 }
diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc
index 1f0b1541d..26be07cff 100644
--- a/src/libexpr/value-to-xml.cc
+++ b/src/libexpr/value-to-xml.cc
@@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
     if (strict) state.forceValue(v);
 
-    switch (v.type) {
+    switch (v.normalType()) {
 
-        case tInt:
+        case nInt:
             doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
             break;
 
-        case tBool:
+        case nBool:
             doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
             break;
 
-        case tString:
+        case nString:
             /* !!! show the context? */
             copyContext(v, context);
             doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
             break;
 
-        case tPath:
+        case nPath:
             doc.writeEmptyElement("path", singletonAttrs("value", v.path));
             break;
 
-        case tNull:
+        case nNull:
             doc.writeEmptyElement("null");
             break;
 
-        case tAttrs:
+        case nAttrs:
             if (state.isDerivation(v)) {
                 XMLAttrs xmlAttrs;
 
@@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
+                    if (a->value->normalType() == nString)
                         xmlAttrs["drvPath"] = drvPath = a->value->string.s;
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
+                    if (a->value->normalType() == nString)
                         xmlAttrs["outPath"] = a->value->string.s;
                 }
 
@@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
             break;
 
-        case tList1: case tList2: case tListN: {
+        case nList: {
             XMLOpenElement _(doc, "list");
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
             break;
         }
 
-        case tLambda: {
+        case nFunction: {
+            if (!v.isLambda()) {
+                // FIXME: Serialize primops and primopapps
+                doc.writeEmptyElement("unevaluated");
+                break;
+            }
             XMLAttrs xmlAttrs;
             if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
             XMLOpenElement _(doc, "function", xmlAttrs);
@@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             break;
         }
 
-        case tExternal:
+        case nExternal:
             v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
             break;
 
-        case tFloat:
+        case nFloat:
             doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
             break;
 
-        default:
+        case nThunk:
             doc.writeEmptyElement("unevaluated");
     }
 }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index a4b5c9e2c..404fd5111 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                                         i.queryName(), j)
                                 });
                             else {
-                                if (v->type == tString) {
+                                if (v->normalType() == nString) {
                                     attrs2["type"] = "string";
                                     attrs2["value"] = v->string.s;
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->type == tInt) {
+                                } else if (v->normalType() == nInt) {
                                     attrs2["type"] = "int";
                                     attrs2["value"] = (format("%1%") % v->integer).str();
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->type == tFloat) {
+                                } else if (v->normalType() == nFloat) {
                                     attrs2["type"] = "float";
                                     attrs2["value"] = (format("%1%") % v->fpoint).str();
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->type == tBool) {
+                                } else if (v->normalType() == nBool) {
                                     attrs2["type"] = "bool";
                                     attrs2["value"] = v->boolean ? "true" : "false";
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->isList()) {
+                                } else if (v->normalType() == nList) {
                                     attrs2["type"] = "strings";
                                     XMLOpenElement m(xml, "meta", attrs2);
                                     for (unsigned int j = 0; j < v->listSize(); ++j) {
-                                        if (v->listElems()[j]->type != tString) continue;
+                                        if (v->listElems()[j]->normalType() != nString) continue;
                                         XMLAttrs attrs3;
                                         attrs3["value"] = v->listElems()[j]->string.s;
                                         xml.writeEmptyElement("string", attrs3);
                                     }
-                              } else if (v->type == tAttrs) {
+                              } else if (v->normalType() == nAttrs) {
                                   attrs2["type"] = "strings";
                                   XMLOpenElement m(xml, "meta", attrs2);
                                   Bindings & attrs = *v->attrs;
                                   for (auto &i : attrs) {
                                       Attr & a(*attrs.find(i.name));
-                                      if(a.value->type != tString) continue;
+                                      if(a.value->normalType() != nString) continue;
                                       XMLAttrs attrs3;
                                       attrs3["type"] = i.name;
                                       attrs3["value"] = a.value->string.s;
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 0f02919de..bba3b1bc6 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -97,10 +97,10 @@ struct CmdEval : MixJSON, InstallableCommand
             recurse = [&](Value & v, const Pos & pos, const Path & path)
             {
                 state->forceValue(v);
-                if (v.type == tString)
+                if (v.normalType() == nString)
                     // FIXME: disallow strings with contexts?
                     writeFile(path, v.string.s);
-                else if (v.type == tAttrs) {
+                else if (v.normalType() == nAttrs) {
                     if (mkdir(path.c_str(), 0777) == -1)
                         throw SysError("creating directory '%s'", path);
                     for (auto & attr : *v.attrs)
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 7a7c71676..80b050091 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -279,7 +279,7 @@ struct CmdFlakeCheck : FlakeCommand
                 if (v.type == tLambda) {
                     if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
                         throw Error("module must match an open attribute set ('{ config, ... }')");
-                } else if (v.type == tAttrs) {
+                } else if (v.normalType() == nAttrs) {
                     for (auto & attr : *v.attrs)
                         try {
                             state->forceValue(*attr.value, *attr.pos);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 3cee81b49..56184efb9 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -446,7 +446,7 @@ bool NixRepl::processLine(string line)
 
         Pos pos;
 
-        if (v.type == tPath || v.type == tString) {
+        if (v.normalType() == nPath || v.normalType() == nString) {
             PathSet context;
             auto filename = state->coerceToString(noPos, v, context);
             pos.file = state->symbols.create(filename);
@@ -669,31 +669,31 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 
     state->forceValue(v);
 
-    switch (v.type) {
+    switch (v.normalType()) {
 
-    case tInt:
+    case nInt:
         str << ANSI_CYAN << v.integer << ANSI_NORMAL;
         break;
 
-    case tBool:
+    case nBool:
         str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
         break;
 
-    case tString:
+    case nString:
         str << ANSI_YELLOW;
         printStringValue(str, v.string.s);
         str << ANSI_NORMAL;
         break;
 
-    case tPath:
+    case nPath:
         str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping?
         break;
 
-    case tNull:
+    case nNull:
         str << ANSI_CYAN "null" ANSI_NORMAL;
         break;
 
-    case tAttrs: {
+    case nAttrs: {
         seen.insert(&v);
 
         bool isDrv = state->isDerivation(v);
@@ -738,9 +738,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
     }
 
-    case tList1:
-    case tList2:
-    case tListN:
+    case nList:
         seen.insert(&v);
 
         str << "[ ";
@@ -761,22 +759,21 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         str << "]";
         break;
 
-    case tLambda: {
-        std::ostringstream s;
-        s << v.lambda.fun->pos;
-        str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
-        break;
-    }
-
-    case tPrimOp:
-        str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
+    case nFunction:
+        if (v.type == tLambda) {
+            std::ostringstream s;
+            s << v.lambda.fun->pos;
+            str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
+        } else if (v.type == tPrimOp) {
+            str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
+        } else if (v.type == tPrimOpApp) {
+            str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
+        } else {
+            abort();
+        }
         break;
 
-    case tPrimOpApp:
-        str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
-        break;
-
-    case tFloat:
+    case nFloat:
         str << v.fpoint;
         break;
 

From bf9890396731a2bbe4f04a49684dee463d818906 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Sat, 12 Dec 2020 02:15:11 +0100
Subject: [PATCH 570/882] Add ValueType checking functions for types that have
 the same NormalType

---
 src/libexpr/eval-inline.hh |  6 +++---
 src/libexpr/eval.cc        | 20 ++++++++++----------
 src/libexpr/flake/flake.cc |  4 ++--
 src/libexpr/primops.cc     |  6 +++---
 src/libexpr/value.hh       | 14 ++++++++++++++
 src/nix/flake.cc           |  6 +++---
 src/nix/main.cc            |  2 +-
 src/nix/repl.cc            |  8 ++++----
 8 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 9b644d5cb..8c40c2565 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -32,7 +32,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
-    if (v.type == tThunk) {
+    if (v.isThunk()) {
         Env * env = v.thunk.env;
         Expr * expr = v.thunk.expr;
         try {
@@ -46,9 +46,9 @@ void EvalState::forceValue(Value & v, const Pos & pos)
             throw;
         }
     }
-    else if (v.type == tApp)
+    else if (v.isApp())
         callFunction(*v.app.left, *v.app.right, v, noPos);
-    else if (v.type == tBlackhole)
+    else if (v.isBlackhole())
         throwEvalError(pos, "infinite recursion encountered");
 }
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f33426b59..5f9d19b8d 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -158,10 +158,10 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 
 const Value *getPrimOp(const Value &v) {
     const Value * primOp = &v;
-    while (primOp->type == tPrimOpApp) {
+    while (primOp->isPrimOpApp()) {
         primOp = primOp->primOpApp.left;
     }
-    assert(primOp->type == tPrimOp);
+    assert(primOp->isPrimOp());
     return primOp;
 }
 
@@ -601,9 +601,9 @@ Value & EvalState::getBuiltin(const string & name)
 
 std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
-    if (v.type == tPrimOp || v.type == tPrimOpApp) {
+    if (v.isPrimOp() || v.isPrimOpApp()) {
         auto v2 = &v;
-        while (v2->type == tPrimOpApp)
+        while (v2->isPrimOpApp())
             v2 = v2->primOpApp.left;
         if (v2->primOp->doc)
             return Doc {
@@ -1227,11 +1227,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     /* Figure out the number of arguments still needed. */
     size_t argsDone = 0;
     Value * primOp = &fun;
-    while (primOp->type == tPrimOpApp) {
+    while (primOp->isPrimOpApp()) {
         argsDone++;
         primOp = primOp->primOpApp.left;
     }
-    assert(primOp->type == tPrimOp);
+    assert(primOp->isPrimOp());
     auto arity = primOp->primOp->arity;
     auto argsLeft = arity - argsDone;
 
@@ -1242,7 +1242,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
         Value * vArgs[arity];
         auto n = arity - 1;
         vArgs[n--] = &arg;
-        for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
+        for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
             vArgs[n--] = arg->primOpApp.right;
 
         /* And call the primop. */
@@ -1264,7 +1264,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     forceValue(fun, pos);
 
-    if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
+    if (fun.isPrimOp() || fun.isPrimOpApp()) {
         callPrimOp(fun, arg, v, pos);
         return;
     }
@@ -1285,7 +1285,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
       }
     }
 
-    if (fun.type != tLambda)
+    if (!fun.isLambda())
         throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
 
     ExprLambda & lambda(*fun.lambda.fun);
@@ -1378,7 +1378,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         }
     }
 
-    if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) {
+    if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
         res = fun;
         return;
     }
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index c126b2c40..2f9658ab8 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -73,7 +73,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
 
 static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
 {
-    if (value.type == tThunk && value.isTrivial())
+    if (value.isThunk() && value.isTrivial())
         state.forceValue(value, pos);
 }
 
@@ -216,7 +216,7 @@ static Flake getFlake(
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
         expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->type == tLambda && outputs->value->lambda.fun->matchAttrs) {
+        if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
             for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d501f7482..f6ca612f4 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2239,11 +2239,11 @@ static RegisterPrimOp primop_catAttrs({
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) {
+    if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) {
         state.mkAttrs(v, 0);
         return;
     }
-    if (args[0]->type != tLambda)
+    if (!args[0]->isLambda())
         throw TypeError({
             .hint = hintfmt("'functionArgs' requires a function"),
             .errPos = pos
@@ -2674,7 +2674,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
            callFunction. */
-        if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
+        if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan)
             return CompareValues()(a, b);
 
         Value vTmp1, vTmp2;
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 0995dcd7b..e743da9c3 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -126,6 +126,20 @@ struct Value
     inline void setExternal() { type = tExternal; };
     inline void setFloat() { type = tFloat; };
 
+    // Functions needed to distinguish the type
+    // These should be removed eventually, by putting the functionality that's
+    // needed by callers into methods of this type
+
+    // normalType() == nThunk
+    inline bool isThunk() const { return type == tThunk; };
+    inline bool isApp() const { return type == tApp; };
+    inline bool isBlackhole() const { return type == tBlackhole; };
+
+    // normalType() == nFunction
+    inline bool isLambda() const { return type == tLambda; };
+    inline bool isPrimOp() const { return type == tPrimOp; };
+    inline bool isPrimOpApp() const { return type == tPrimOpApp; };
+
     union
     {
         NixInt integer;
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 80b050091..e4da0348c 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -260,7 +260,7 @@ struct CmdFlakeCheck : FlakeCommand
         auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) {
             try {
                 state->forceValue(v, pos);
-                if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
+                if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
                     throw Error("overlay does not take an argument named 'final'");
                 auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
                 if (!body || body->matchAttrs || std::string(body->arg) != "prev")
@@ -276,7 +276,7 @@ struct CmdFlakeCheck : FlakeCommand
         auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) {
             try {
                 state->forceValue(v, pos);
-                if (v.type == tLambda) {
+                if (v.isLambda()) {
                     if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
                         throw Error("module must match an open attribute set ('{ config, ... }')");
                 } else if (v.normalType() == nAttrs) {
@@ -371,7 +371,7 @@ struct CmdFlakeCheck : FlakeCommand
         auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) {
             try {
                 state->forceValue(v, pos);
-                if (v.type != tLambda)
+                if (!v.isLambda())
                     throw Error("bundler must be a function");
                 if (!v.lambda.fun->formals ||
                     v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() ||
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 27b1d7257..e7a15dec9 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -272,7 +272,7 @@ void mainWrapped(int argc, char * * argv)
         auto builtins = state.baseEnv.values[0]->attrs;
         for (auto & builtin : *builtins) {
             auto b = nlohmann::json::object();
-            if (builtin.value->type != tPrimOp) continue;
+            if (!builtin.value->isPrimOp()) continue;
             auto primOp = builtin.value->primOp;
             if (!primOp->doc) continue;
             b["arity"] = primOp->arity;
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 56184efb9..047e2dc59 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -450,7 +450,7 @@ bool NixRepl::processLine(string line)
             PathSet context;
             auto filename = state->coerceToString(noPos, v, context);
             pos.file = state->symbols.create(filename);
-        } else if (v.type == tLambda) {
+        } else if (v.isLambda()) {
             pos = v.lambda.fun->pos;
         } else {
             // assume it's a derivation
@@ -760,13 +760,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case nFunction:
-        if (v.type == tLambda) {
+        if (v.isLambda()) {
             std::ostringstream s;
             s << v.lambda.fun->pos;
             str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
-        } else if (v.type == tPrimOp) {
+        } else if (v.isPrimOp()) {
             str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
-        } else if (v.type == tPrimOpApp) {
+        } else if (v.isPrimOpApp()) {
             str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
         } else {
             abort();

From 730b152b190135adef2f53c7a80cfd1111d37ead Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Sat, 12 Dec 2020 02:22:58 +0100
Subject: [PATCH 571/882] Make Value::type private

This is an implementation detail and shouldn't be used. Use normalType()
and the various is<Type> functions instead
---
 src/libexpr/eval.cc  | 2 +-
 src/libexpr/value.hh | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 5f9d19b8d..1d11039ad 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -68,7 +68,7 @@ RootValue allocRootValue(Value * v)
 }
 
 
-static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
+void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
 
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index e743da9c3..4050d7e4b 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -106,8 +106,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 
 struct Value
 {
+private:
     ValueType type;
 
+friend std::string showType(const Value & v);
+friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
+
+public:
+
     inline void setInt() { type = tInt; };
     inline void setBool() { type = tBool; };
     inline void setString() { type = tString; };

From f890830b337f321f17ad36b45d7d63801a753554 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 11 Dec 2020 16:31:14 +0100
Subject: [PATCH 572/882] primops/fromJSON: add error position in case of parse
 error

This makes it easier to track down where invalid JSON was passed to
`builtins.fromJSON`.
---
 src/libexpr/primops.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 41f06c219..63624c520 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1621,7 +1621,12 @@ static RegisterPrimOp primop_toJSON({
 static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string s = state.forceStringNoCtx(*args[0], pos);
-    parseJSON(state, s, v);
+    try {
+        parseJSON(state, s, v);
+    } catch (JSONParseError &e) {
+        e.addTrace(pos, "while decoding a JSON string");
+        throw e;
+    }
 }
 
 static RegisterPrimOp primop_fromJSON({

From 44c3fbc6e03ec518f6174c2b7c21b603973beb91 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 20 Oct 2020 15:03:54 +0200
Subject: [PATCH 573/882] Fix `addTextToStore` for binary caches

Because of a too eager refactoring, `addTextToStore` used to throw an
error because the input wasn't a valid nar.

Partially revert that refactoring to wrap the text into a proper nar
(using `dumpString`) to make this method work again
---
 src/libstore/binary-cache-store.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 5b081c1ae..94c11355f 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -433,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     if (!repair && isValidPath(path))
         return path;
 
-    auto source = StringSource { s };
+    StringSink sink;
+    dumpString(s, sink);
+    auto source = StringSource { *sink.s };
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info { path, nar.first };
         info.narSize = nar.second;

From 7080321618e29033a8b5dc2f9fc938dcf2df270d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 15 Dec 2020 10:54:24 +0100
Subject: [PATCH 574/882] Use the fs accessor for readInvalidDerivation

Extend `FSAccessor::readFile` to allow not checking that the path is a
valid one, and rewrite `readInvalidDerivation` using this extended
`readFile`.

Several places in the code use `readInvalidDerivation`, either because
they need to read a derivation that has been written in the store but
not registered yet, or more generally to prevent a deadlock because
`readDerivation` tries to lock the state, so can't be called from a
place where the lock is already held.
However, `readInvalidDerivation` implicitely assumes that the store is a
`LocalFSStore`, which isn't always the case.

The concrete motivation for this is that it's required for `nix copy
--from someBinaryCache` to work, which is tremendously useful for the
tests.
---
 src/libstore/fs-accessor.hh        |  9 ++++++++-
 src/libstore/local-fs-store.cc     |  8 ++++----
 src/libstore/nar-accessor.cc       |  2 +-
 src/libstore/remote-fs-accessor.cc |  8 ++++----
 src/libstore/remote-fs-accessor.hh |  4 ++--
 src/libstore/store-api.cc          | 21 +++++++++------------
 6 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index 64780a6da..c825e84f2 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -25,7 +25,14 @@ public:
 
     virtual StringSet readDirectory(const Path & path) = 0;
 
-    virtual std::string readFile(const Path & path) = 0;
+    /**
+     * Read a file inside the store.
+     *
+     * If `requireValidPath` is set to `true` (the default), the path must be
+     * inside a valid store path, otherwise it just needs to be physically
+     * present (but not necessarily properly registered)
+     */
+    virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
 
     virtual std::string readLink(const Path & path) = 0;
 };
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index e7c3dae92..6de13c73a 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor
 
     LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { }
 
-    Path toRealPath(const Path & path)
+    Path toRealPath(const Path & path, bool requireValidPath = true)
     {
         auto storePath = store->toStorePath(path).first;
-        if (!store->isValidPath(storePath))
+        if (requireValidPath && !store->isValidPath(storePath))
             throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
@@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor
         return res;
     }
 
-    std::string readFile(const Path & path) override
+    std::string readFile(const Path & path, bool requireValidPath = true) override
     {
-        return nix::readFile(toRealPath(path));
+        return nix::readFile(toRealPath(path, requireValidPath));
     }
 
     std::string readLink(const Path & path) override
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 1427a0f98..784ebb719 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor
         return res;
     }
 
-    std::string readFile(const Path & path) override
+    std::string readFile(const Path & path, bool requireValidPath = true) override
     {
         auto i = get(path);
         if (i.type != FSAccessor::Type::tRegular)
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 63bde92de..f43456f0b 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string &
     }
 }
 
-std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
+std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
 {
     auto path = canonPath(path_);
 
     auto [storePath, restPath] = store->toStorePath(path);
 
-    if (!store->isValidPath(storePath))
+    if (requireValidPath && !store->isValidPath(storePath))
         throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
 
     auto i = nars.find(std::string(storePath.hashPart()));
@@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path)
     return res.first->readDirectory(res.second);
 }
 
-std::string RemoteFSAccessor::readFile(const Path & path)
+std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath)
 {
-    auto res = fetch(path);
+    auto res = fetch(path, requireValidPath);
     return res.first->readFile(res.second);
 }
 
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 347cf5764..594852d0e 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor
 
     Path cacheDir;
 
-    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
+    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true);
 
     friend class BinaryCacheStore;
 
@@ -32,7 +32,7 @@ public:
 
     StringSet readDirectory(const Path & path) override;
 
-    std::string readFile(const Path & path) override;
+    std::string readFile(const Path & path, bool requireValidPath = true) override;
 
     std::string readLink(const Path & path) override;
 };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7bf9235b2..25e28cffa 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1018,26 +1018,23 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
     return readDerivation(drvPath);
 }
 
-
-Derivation Store::readDerivation(const StorePath & drvPath)
+Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath)
 {
-    auto accessor = getFSAccessor();
+    auto accessor = store.getFSAccessor();
     try {
-        return parseDerivation(*this,
-            accessor->readFile(printStorePath(drvPath)),
+        return parseDerivation(store,
+            accessor->readFile(store.printStorePath(drvPath), requireValidPath),
             Derivation::nameFromPath(drvPath));
     } catch (FormatError & e) {
-        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
+        throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());
     }
 }
 
+Derivation Store::readDerivation(const StorePath & drvPath)
+{ return readDerivationCommon(*this, drvPath, true); }
+
 Derivation Store::readInvalidDerivation(const StorePath & drvPath)
-{
-    return parseDerivation(
-        *this,
-        readFile(Store::toRealPath(drvPath)),
-        Derivation::nameFromPath(drvPath));
-}
+{ return readDerivationCommon(*this, drvPath, false); }
 
 }
 

From 6e899278d305da904fb766937f56344841c022b3 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 6 Nov 2020 09:51:17 +0100
Subject: [PATCH 575/882] Better detect when `buildPaths` would be a no-op
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`buildPaths` can be called even for stores where it's not defined in case it's
bound to be a no-op.
The “no-op detection” mechanism was only detecting the case wher `buildPaths`
was called on a set of (non-drv) paths that were already present on the store.

This commit extends this mechanism to also detect the case where `buildPaths`
is called on a set of derivation outputs which are already built on the store.

This only works with the ca-derivations flag. It could be possible to
extend this to also work without it, but it would add quite a bit of
complexity, and it's not used without it anyways.
---
 src/libstore/store-api.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7bf9235b2..50905bb33 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -729,9 +729,17 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
     StorePathSet paths2;
 
     for (auto & path : paths) {
-        if (path.path.isDerivation())
-            unsupported("buildPaths");
-        paths2.insert(path.path);
+        if (path.path.isDerivation()) {
+            if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                for (auto & outputName : path.outputs) {
+                    if (!queryRealisation({path.path, outputName}))
+                        unsupported("buildPaths");
+                }
+            } else
+                unsupported("buildPaths");
+
+        } else
+            paths2.insert(path.path);
     }
 
     if (queryValidPaths(paths2).size() != paths2.size())

From 962b82ef25069893779ed56d31e44814793f9273 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 15 Dec 2020 09:34:45 +0100
Subject: [PATCH 576/882] Fix BinaryCacheStore::registerDrvOutput

Was crashing because coercing a json document into a string is only
valid if the json is a string, otherwise we need to call `.dump()`
---
 src/libstore/binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 94c11355f..4f5f8607d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -460,7 +460,7 @@ std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOut
 
 void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
     auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
-    upsertFile(filePath, info.toJSON(), "application/json");
+    upsertFile(filePath, info.toJSON().dump(), "application/json");
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()

From cac8d5b742ec0cb80ad7232e20f63c74a217e545 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 16 Dec 2020 13:36:17 +0100
Subject: [PATCH 577/882] Don't ignore an absent drv file in
 queryPartialDrvOutputMap

This ignore was here because `queryPartialDrvOutputMap` was used both
1. as a cache to avoid having to re-read the derivation (when gc-ing for
example), and
2. as the source of truth for ca realisations

The use-case 2. required it to be able to work even when the derivation
wasn't there anymore (see https://github.com/NixOS/nix/issues/4138).
However, this use-case is now handled by `queryRealisation`, meaning
that we can safely error out if the derivation isn't there anymore
---
 src/libstore/local-store.cc | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1539c94e2..20bbc73cf 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -912,12 +912,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
         auto state(_state.lock());
         std::map<std::string, std::optional<StorePath>> outputs;
         uint64_t drvId;
-        try {
-            drvId = queryValidPathId(*state, path);
-        } catch (InvalidPath&) {
-            // Ignore non-existing drvs as they might still have an output map
-            // defined if ca-derivations is enabled
-        }
+        drvId = queryValidPathId(*state, path);
         auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
         while (use.next())
             outputs.insert_or_assign(

From 4d458394991f3086c3c9c306d000e6c0058c4fa7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 17 Dec 2020 11:35:24 +0100
Subject: [PATCH 578/882] Fix the detection of already built drv outputs

PRs #4370 and #4348 had a bad interaction in that the second broke the fist
one in a not trivial way.

The issue was that since #4348 the logic for detecting whether a
derivation output is already built requires some logic that was specific
to the `LocalStore`.

It happens though that most of this logic could be upstreamed to any `Store`,
which is what this commit does.
---
 src/libstore/derivations.cc | 32 +++++++++++++++++++++++++-
 src/libstore/derivations.hh |  4 ++++
 src/libstore/local-store.cc | 45 ++++---------------------------------
 src/libstore/local-store.hh |  2 +-
 src/libstore/store-api.cc   | 39 +++++++++++++++++++++++++-------
 src/libstore/store-api.hh   |  9 ++++++--
 6 files changed, 78 insertions(+), 53 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 5bcc7f012..7466c7d41 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
+std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
     BasicDerivation resolved { *this };
 
     // Input paths that we'll want to rewrite in the derivation
@@ -771,4 +771,34 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
     return resolved;
 }
 
+std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
+{
+    auto drvPath = writeDerivation(store, *this, NoRepair, false);
+    return Derivation::tryResolve(store, drvPath);
+}
+
+std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
+{
+    // This is quite dirty and leaky, but will disappear once #4340 is merged
+    static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
+
+    {
+        auto resolutions = resolutionsCache.lock();
+        auto resolvedDrvIter = resolutions->find(drvPath);
+        if (resolvedDrvIter != resolutions->end()) {
+            auto & [_, resolvedDrv] = *resolvedDrvIter;
+                return *resolvedDrv;
+        }
+    }
+
+    /* Try resolve drv and use that path instead. */
+    auto drv = store.readDerivation(drvPath);
+    auto attempt = drv.tryResolveUncached(store);
+    if (!attempt)
+        return std::nullopt;
+    /* Store in memo table. */
+    resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
+    return *attempt;
+}
+
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 4e5985fab..3d8f19aef 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -138,10 +138,14 @@ struct Derivation : BasicDerivation
 
        2. Input placeholders are replaced with realized input store paths. */
     std::optional<BasicDerivation> tryResolve(Store & store);
+    static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
 
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
+
+private:
+    std::optional<BasicDerivation> tryResolveUncached(Store & store);
 };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 20bbc73cf..e9f9bde4d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -877,35 +877,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
     });
 }
 
-// Try to resolve the derivation at path `original`, with a caching layer
-// to make it more efficient
-std::optional<Derivation> cachedResolve(
-    LocalStore& store,
-    const StorePath& original)
-{
-    // This is quite dirty and leaky, but will disappear once #4340 is merged
-    static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
-    {
-        auto resolutions = resolutionsCache.lock();
-        auto resolvedDrvIter = resolutions->find(original);
-        if (resolvedDrvIter != resolutions->end()) {
-            auto & [_, resolvedDrv] = *resolvedDrvIter;
-                return *resolvedDrv;
-        }
-    }
-
-    /* Try resolve drv and use that path instead. */
-    auto drv = store.readDerivation(original);
-    auto attempt = drv.tryResolve(store);
-    if (!attempt)
-        return std::nullopt;
-    /* Store in memo table. */
-    resolutionsCache.lock()->insert_or_assign(original, *attempt);
-    return *attempt;
-}
 
 std::map<std::string, std::optional<StorePath>>
-LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
+LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
 {
     auto path = path_;
     auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
@@ -924,20 +898,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
     if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
         return outputs;
 
-    auto drv = readDerivation(path);
-
-    auto resolvedDrv = cachedResolve(*this, path);
-
-    if (!resolvedDrv) {
-        for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) {
-            if (!outputs.count(outputName))
-                outputs.emplace(outputName, std::nullopt);
-        }
-        return outputs;
-    }
-
-    auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv);
-    for (auto& [outputName, hash] : resolvedDrvHashes) {
+    auto drv = readInvalidDerivation(path);
+    auto drvHashes = staticOutputHashes(*this, drv);
+    for (auto& [outputName, hash] : drvHashes) {
         auto realisation = queryRealisation(DrvOutput{hash, outputName});
         if (realisation)
             outputs.insert_or_assign(outputName, realisation->outPath);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 69559e346..877dba742 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -127,7 +127,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 50905bb33..2cd39ab11 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -366,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
+std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapNoResolve(const StorePath & path)
+{
+    std::map<std::string, std::optional<StorePath>> outputs;
+    auto drv = readInvalidDerivation(path);
+    for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) {
+        outputs.emplace(outputName, output.second);
+    }
+    return outputs;
+}
+
+std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
+{
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        auto resolvedDrv = Derivation::tryResolve(*this, path);
+        if (resolvedDrv) {
+            auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true);
+            if (isValidPath(resolvedDrvPath))
+                return queryDerivationOutputMapNoResolve(resolvedDrvPath);
+        }
+    }
+    return queryDerivationOutputMapNoResolve(path);
+}
+
 OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
     auto resp = queryPartialDerivationOutputMap(path);
     OutputPathMap result;
@@ -730,14 +753,14 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
 
     for (auto & path : paths) {
         if (path.path.isDerivation()) {
-            if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-                for (auto & outputName : path.outputs) {
-                    if (!queryRealisation({path.path, outputName}))
-                        unsupported("buildPaths");
-                }
-            } else
-                unsupported("buildPaths");
-
+            auto outPaths = queryPartialDerivationOutputMap(path.path);
+            for (auto & outputName : path.outputs) {
+                auto currentOutputPathIter = outPaths.find(outputName);
+                if (currentOutputPathIter == outPaths.end() ||
+                    !currentOutputPathIter->second ||
+                    !isValidPath(*currentOutputPathIter->second))
+                    unsupported("buildPaths");
+            }
         } else
             paths2.insert(path.path);
     }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 7cdadc1f3..ce95b78b1 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -416,8 +416,13 @@ public:
     /* Query the mapping outputName => outputPath for the given derivation. All
        outputs are mentioned so ones mising the mapping are mapped to
        `std::nullopt`.  */
-    virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path)
-    { unsupported("queryPartialDerivationOutputMap"); }
+    virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
+
+    /*
+     * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve
+     * the derivation
+     */
+    virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path);
 
     /* Query the mapping outputName=>outputPath for the given derivation.
        Assume every output has a mapping and throw an exception otherwise. */

From d67e02919c7f941615407dfd14cfdab6a28c4c26 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Thu, 17 Dec 2020 14:42:52 +0100
Subject: [PATCH 579/882] Rename ValueType -> InternalType, NormalType ->
 ValueType

And Value::type to Value::internalType, such that type() can be used in
the next commit to get the new ValueType
---
 src/libexpr/eval.cc        | 16 +++++-----
 src/libexpr/eval.hh        |  2 +-
 src/libexpr/flake/flake.cc |  2 +-
 src/libexpr/value.hh       | 64 +++++++++++++++++++-------------------
 4 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 1d11039ad..e14eb01c7 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -77,7 +77,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         return;
     }
 
-    switch (v.type) {
+    switch (v.internalType) {
     case tInt:
         str << v.integer;
         break;
@@ -165,7 +165,7 @@ const Value *getPrimOp(const Value &v) {
     return primOp;
 }
 
-string showType(NormalType type)
+string showType(ValueType type)
 {
     switch (type) {
         case nInt: return "an integer";
@@ -186,7 +186,7 @@ string showType(NormalType type)
 
 string showType(const Value & v)
 {
-    switch (v.type) {
+    switch (v.internalType) {
         case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
             return fmt("the built-in function '%s'", string(v.primOp->name));
@@ -205,9 +205,9 @@ string showType(const Value & v)
 bool Value::isTrivial() const
 {
     return
-        type != tApp
-        && type != tPrimOpApp
-        && (type != tThunk
+        internalType != tApp
+        && internalType != tPrimOpApp
+        && (internalType != tThunk
             || (dynamic_cast<ExprAttrs *>(thunk.expr)
                 && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
             || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -1562,7 +1562,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     NixFloat nf = 0;
 
     bool first = !forceString;
-    NormalType firstType = nString;
+    ValueType firstType = nString;
 
     for (auto & i : *es) {
         Value vTmp;
@@ -1728,7 +1728,7 @@ void copyContext(const Value & v, PathSet & context)
 std::vector<std::pair<Path, std::string>> Value::getContext()
 {
     std::vector<std::pair<Path, std::string>> res;
-    assert(type == tString);
+    assert(internalType == tString);
     if (string.context)
         for (const char * * p = string.context; *p; ++p)
             res.push_back(decodeContext(*p));
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 211529954..0e1f61baa 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -346,7 +346,7 @@ private:
 
 
 /* Return a string representing the type of the value `v'. */
-string showType(NormalType type);
+string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 2f9658ab8..987e7e24b 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -78,7 +78,7 @@ static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
 }
 
 
-static void expectType(EvalState & state, NormalType type,
+static void expectType(EvalState & state, ValueType type,
     Value & value, const Pos & pos)
 {
     forceTrivialValue(state, value, pos);
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 4050d7e4b..8b312bf03 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -27,7 +27,7 @@ typedef enum {
     tPrimOpApp,
     tExternal,
     tFloat
-} ValueType;
+} InternalType;
 
 // This type abstracts over all actual value types in the language,
 // grouping together implementation details like tList*, different function
@@ -44,7 +44,7 @@ typedef enum {
     nList,
     nFunction,
     nExternal
-} NormalType;
+} ValueType;
 
 class Bindings;
 struct Env;
@@ -107,44 +107,44 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 struct Value
 {
 private:
-    ValueType type;
+    InternalType internalType;
 
 friend std::string showType(const Value & v);
 friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
 
 public:
 
-    inline void setInt() { type = tInt; };
-    inline void setBool() { type = tBool; };
-    inline void setString() { type = tString; };
-    inline void setPath() { type = tPath; };
-    inline void setNull() { type = tNull; };
-    inline void setAttrs() { type = tAttrs; };
-    inline void setList1() { type = tList1; };
-    inline void setList2() { type = tList2; };
-    inline void setListN() { type = tListN; };
-    inline void setThunk() { type = tThunk; };
-    inline void setApp() { type = tApp; };
-    inline void setLambda() { type = tLambda; };
-    inline void setBlackhole() { type = tBlackhole; };
-    inline void setPrimOp() { type = tPrimOp; };
-    inline void setPrimOpApp() { type = tPrimOpApp; };
-    inline void setExternal() { type = tExternal; };
-    inline void setFloat() { type = tFloat; };
+    inline void setInt() { internalType = tInt; };
+    inline void setBool() { internalType = tBool; };
+    inline void setString() { internalType = tString; };
+    inline void setPath() { internalType = tPath; };
+    inline void setNull() { internalType = tNull; };
+    inline void setAttrs() { internalType = tAttrs; };
+    inline void setList1() { internalType = tList1; };
+    inline void setList2() { internalType = tList2; };
+    inline void setListN() { internalType = tListN; };
+    inline void setThunk() { internalType = tThunk; };
+    inline void setApp() { internalType = tApp; };
+    inline void setLambda() { internalType = tLambda; };
+    inline void setBlackhole() { internalType = tBlackhole; };
+    inline void setPrimOp() { internalType = tPrimOp; };
+    inline void setPrimOpApp() { internalType = tPrimOpApp; };
+    inline void setExternal() { internalType = tExternal; };
+    inline void setFloat() { internalType = tFloat; };
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
     // needed by callers into methods of this type
 
     // normalType() == nThunk
-    inline bool isThunk() const { return type == tThunk; };
-    inline bool isApp() const { return type == tApp; };
-    inline bool isBlackhole() const { return type == tBlackhole; };
+    inline bool isThunk() const { return internalType == tThunk; };
+    inline bool isApp() const { return internalType == tApp; };
+    inline bool isBlackhole() const { return internalType == tBlackhole; };
 
     // normalType() == nFunction
-    inline bool isLambda() const { return type == tLambda; };
-    inline bool isPrimOp() const { return type == tPrimOp; };
-    inline bool isPrimOpApp() const { return type == tPrimOpApp; };
+    inline bool isLambda() const { return internalType == tLambda; };
+    inline bool isPrimOp() const { return internalType == tPrimOp; };
+    inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
 
     union
     {
@@ -204,9 +204,9 @@ public:
 
     // Returns the normal type of a Value. This only returns nThunk if the
     // Value hasn't been forceValue'd
-    inline NormalType normalType() const
+    inline ValueType normalType() const
     {
-        switch (type) {
+        switch (internalType) {
             case tInt: return nInt;
             case tBool: return nBool;
             case tString: return nString;
@@ -224,22 +224,22 @@ public:
 
     bool isList() const
     {
-        return type == tList1 || type == tList2 || type == tListN;
+        return internalType == tList1 || internalType == tList2 || internalType == tListN;
     }
 
     Value * * listElems()
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
 
     const Value * const * listElems() const
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
 
     size_t listSize() const
     {
-        return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
+        return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size;
     }
 
     /* Check whether forcing this value requires a trivial amount of

From 12e65078ef5c511196c9e48f7fdf71f6c0e5c89f Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Thu, 17 Dec 2020 14:45:45 +0100
Subject: [PATCH 580/882] Rename Value::normalType() -> Value::type()

---
 src/libexpr/attr-path.cc              |  2 +-
 src/libexpr/eval-cache.cc             | 26 +++++-----
 src/libexpr/eval-inline.hh            |  4 +-
 src/libexpr/eval.cc                   | 74 +++++++++++++--------------
 src/libexpr/flake/flake.cc            | 16 +++---
 src/libexpr/get-drvs.cc               | 30 +++++------
 src/libexpr/primops.cc                | 44 ++++++++--------
 src/libexpr/primops/fetchMercurial.cc |  2 +-
 src/libexpr/primops/fetchTree.cc      | 12 ++---
 src/libexpr/value-to-json.cc          |  2 +-
 src/libexpr/value-to-xml.cc           |  6 +--
 src/libexpr/value.hh                  |  6 +--
 src/nix-env/nix-env.cc                | 16 +++---
 src/nix/eval.cc                       |  4 +-
 src/nix/flake.cc                      |  2 +-
 src/nix/repl.cc                       |  4 +-
 16 files changed, 125 insertions(+), 125 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 54e13e6a2..2d37dcb7e 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -67,7 +67,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
         if (apType == apAttr) {
 
-            if (v->normalType() != nAttrs)
+            if (v->type() != nAttrs)
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a set but is %2%",
                     attrPath,
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 3c97f1201..75e9af787 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -390,14 +390,14 @@ Value & AttrCursor::forceValue()
     }
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
-        if (v.normalType() == nString)
+        if (v.type() == nString)
             cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
                            string_t{v.string.s, {}}};
-        else if (v.normalType() == nPath)
+        else if (v.type() == nPath)
             cachedValue = {root->db->setString(getKey(), v.path), v.path};
-        else if (v.normalType() == nBool)
+        else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
-        else if (v.normalType() == nAttrs)
+        else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
             cachedValue = {root->db->setMisc(getKey()), misc_t()};
@@ -442,7 +442,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
 
     auto & v = forceValue();
 
-    if (v.normalType() != nAttrs)
+    if (v.type() != nAttrs)
         return nullptr;
         //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
@@ -512,10 +512,10 @@ std::string AttrCursor::getString()
 
     auto & v = forceValue();
 
-    if (v.normalType() != nString && v.normalType() != nPath)
-        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType()));
+    if (v.type() != nString && v.type() != nPath)
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
 
-    return v.normalType() == nString ? v.string.s : v.path;
+    return v.type() == nString ? v.string.s : v.path;
 }
 
 string_t AttrCursor::getStringWithContext()
@@ -543,12 +543,12 @@ string_t AttrCursor::getStringWithContext()
 
     auto & v = forceValue();
 
-    if (v.normalType() == nString)
+    if (v.type() == nString)
         return {v.string.s, v.getContext()};
-    else if (v.normalType() == nPath)
+    else if (v.type() == nPath)
         return {v.path, {}};
     else
-        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.normalType()));
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
 }
 
 bool AttrCursor::getBool()
@@ -567,7 +567,7 @@ bool AttrCursor::getBool()
 
     auto & v = forceValue();
 
-    if (v.normalType() != nBool)
+    if (v.type() != nBool)
         throw TypeError("'%s' is not a Boolean", getAttrPathStr());
 
     return v.boolean;
@@ -589,7 +589,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
 
     auto & v = forceValue();
 
-    if (v.normalType() != nAttrs)
+    if (v.type() != nAttrs)
         throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
     std::vector<Symbol> attrs;
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 8c40c2565..e56ce261c 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -56,7 +56,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
 inline void EvalState::forceAttrs(Value & v)
 {
     forceValue(v);
-    if (v.normalType() != nAttrs)
+    if (v.type() != nAttrs)
         throwTypeError("value is %1% while a set was expected", v);
 }
 
@@ -64,7 +64,7 @@ inline void EvalState::forceAttrs(Value & v)
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.normalType() != nAttrs)
+    if (v.type() != nAttrs)
         throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e14eb01c7..2f8d6d259 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -197,7 +197,7 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tBlackhole: return "a black hole";
     default:
-        return showType(v.normalType());
+        return showType(v.type());
     }
 }
 
@@ -947,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e)
 {
     Value v;
     e->eval(*this, env, v);
-    if (v.normalType() != nBool)
+    if (v.type() != nBool)
         throwTypeError("value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -957,7 +957,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
 {
     Value v;
     e->eval(*this, env, v);
-    if (v.normalType() != nBool)
+    if (v.type() != nBool)
         throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -966,7 +966,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
 inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v)
 {
     e->eval(*this, env, v);
-    if (v.normalType() != nAttrs)
+    if (v.type() != nAttrs)
         throwTypeError("value is %1% while a set was expected", v);
 }
 
@@ -1066,7 +1066,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Value nameVal;
         i.nameExpr->eval(state, *dynamicEnv, nameVal);
         state.forceValue(nameVal, i.pos);
-        if (nameVal.normalType() == nNull)
+        if (nameVal.type() == nNull)
             continue;
         state.forceStringNoCtx(nameVal);
         Symbol nameSym = state.symbols.create(nameVal.string.s);
@@ -1151,7 +1151,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             Symbol name = getName(i, state, env);
             if (def) {
                 state.forceValue(*vAttrs, pos);
-                if (vAttrs->normalType() != nAttrs ||
+                if (vAttrs->type() != nAttrs ||
                     (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
                 {
                     def->eval(state, env, v);
@@ -1191,7 +1191,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
         state.forceValue(*vAttrs);
         Bindings::iterator j;
         Symbol name = getName(i, state, env);
-        if (vAttrs->normalType() != nAttrs ||
+        if (vAttrs->type() != nAttrs ||
             (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
         {
             mkBool(v, false);
@@ -1269,7 +1269,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         return;
     }
 
-    if (fun.normalType() == nAttrs) {
+    if (fun.type() == nAttrs) {
       auto found = fun.attrs->find(sFunctor);
       if (found != fun.attrs->end()) {
         /* fun may be allocated on the stack of the calling function,
@@ -1368,7 +1368,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
     forceValue(fun);
 
-    if (fun.normalType() == nAttrs) {
+    if (fun.type() == nAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
             Value * v = allocValue();
@@ -1573,14 +1573,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
-            firstType = vTmp.normalType();
+            firstType = vTmp.type();
             first = false;
         }
 
         if (firstType == nInt) {
-            if (vTmp.normalType() == nInt) {
+            if (vTmp.type() == nInt) {
                 n += vTmp.integer;
-            } else if (vTmp.normalType() == nFloat) {
+            } else if (vTmp.type() == nFloat) {
                 // Upgrade the type from int to float;
                 firstType = nFloat;
                 nf = n;
@@ -1588,9 +1588,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
             } else
                 throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == nFloat) {
-            if (vTmp.normalType() == nInt) {
+            if (vTmp.type() == nInt) {
                 nf += vTmp.integer;
-            } else if (vTmp.normalType() == nFloat) {
+            } else if (vTmp.type() == nFloat) {
                 nf += vTmp.fpoint;
             } else
                 throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
@@ -1629,7 +1629,7 @@ void EvalState::forceValueDeep(Value & v)
 
         forceValue(v);
 
-        if (v.normalType() == nAttrs) {
+        if (v.type() == nAttrs) {
             for (auto & i : *v.attrs)
                 try {
                     recurse(*i.value);
@@ -1652,7 +1652,7 @@ void EvalState::forceValueDeep(Value & v)
 NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.normalType() != nInt)
+    if (v.type() != nInt)
         throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
@@ -1661,9 +1661,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.normalType() == nInt)
+    if (v.type() == nInt)
         return v.integer;
-    else if (v.normalType() != nFloat)
+    else if (v.type() != nFloat)
         throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
@@ -1672,7 +1672,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.normalType() != nBool)
+    if (v.type() != nBool)
         throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -1680,14 +1680,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 
 bool EvalState::isFunctor(Value & fun)
 {
-    return fun.normalType() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
+    return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
 }
 
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.normalType() != nFunction && !isFunctor(v))
+    if (v.type() != nFunction && !isFunctor(v))
         throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
@@ -1695,7 +1695,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 string EvalState::forceString(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.normalType() != nString) {
+    if (v.type() != nString) {
         if (pos)
             throwTypeError(pos, "value is %1% while a string was expected", v);
         else
@@ -1761,11 +1761,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
 
 bool EvalState::isDerivation(Value & v)
 {
-    if (v.normalType() != nAttrs) return false;
+    if (v.type() != nAttrs) return false;
     Bindings::iterator i = v.attrs->find(sType);
     if (i == v.attrs->end()) return false;
     forceValue(*i->value);
-    if (i->value->normalType() != nString) return false;
+    if (i->value->type() != nString) return false;
     return strcmp(i->value->string.s, "derivation") == 0;
 }
 
@@ -1790,17 +1790,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
     string s;
 
-    if (v.normalType() == nString) {
+    if (v.type() == nString) {
         copyContext(v, context);
         return v.string.s;
     }
 
-    if (v.normalType() == nPath) {
+    if (v.type() == nPath) {
         Path path(canonPath(v.path));
         return copyToStore ? copyPathToStore(context, path) : path;
     }
 
-    if (v.normalType() == nAttrs) {
+    if (v.type() == nAttrs) {
         auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
         if (maybeString) {
             return *maybeString;
@@ -1810,18 +1810,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
-    if (v.normalType() == nExternal)
+    if (v.type() == nExternal)
         return v.external->coerceToString(pos, context, coerceMore, copyToStore);
 
     if (coerceMore) {
 
         /* Note that `false' is represented as an empty string for
            shell scripting convenience, just like `null'. */
-        if (v.normalType() == nBool && v.boolean) return "1";
-        if (v.normalType() == nBool && !v.boolean) return "";
-        if (v.normalType() == nInt) return std::to_string(v.integer);
-        if (v.normalType() == nFloat) return std::to_string(v.fpoint);
-        if (v.normalType() == nNull) return "";
+        if (v.type() == nBool && v.boolean) return "1";
+        if (v.type() == nBool && !v.boolean) return "";
+        if (v.type() == nInt) return std::to_string(v.integer);
+        if (v.type() == nFloat) return std::to_string(v.fpoint);
+        if (v.type() == nNull) return "";
 
         if (v.isList()) {
             string result;
@@ -1884,15 +1884,15 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     if (&v1 == &v2) return true;
 
     // Special case type-compatibility between float and int
-    if (v1.normalType() == nInt && v2.normalType() == nFloat)
+    if (v1.type() == nInt && v2.type() == nFloat)
         return v1.integer == v2.fpoint;
-    if (v1.normalType() == nFloat && v2.normalType() == nInt)
+    if (v1.type() == nFloat && v2.type() == nInt)
         return v1.fpoint == v2.integer;
 
     // All other types are not compatible with each other.
-    if (v1.normalType() != v2.normalType()) return false;
+    if (v1.type() != v2.type()) return false;
 
-    switch (v1.normalType()) {
+    switch (v1.type()) {
 
         case nInt:
             return v1.integer == v2.integer;
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 987e7e24b..4f021570c 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -82,9 +82,9 @@ static void expectType(EvalState & state, ValueType type,
     Value & value, const Pos & pos)
 {
     forceTrivialValue(state, value, pos);
-    if (value.normalType() != type)
+    if (value.type() != type)
         throw Error("expected %s but got %s at %s",
-            showType(type), showType(value.normalType()), pos);
+            showType(type), showType(value.type()), pos);
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
@@ -120,7 +120,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
-                if (attr.value->normalType() == nString)
+                if (attr.value->type() == nString)
                     attrs.emplace(attr.name, attr.value->string.s);
                 else
                     throw TypeError("flake input attribute '%s' is %s while a string is expected",
@@ -235,17 +235,17 @@ static Flake getFlake(
 
         for (auto & setting : *nixConfig->value->attrs) {
             forceTrivialValue(state, *setting.value, *setting.pos);
-            if (setting.value->normalType() == nString)
+            if (setting.value->type() == nString)
                 flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
-            else if (setting.value->normalType() == nInt)
+            else if (setting.value->type() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
-            else if (setting.value->normalType() == nBool)
+            else if (setting.value->type() == nBool)
                 flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
-            else if (setting.value->normalType() == nList) {
+            else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
                 for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
                     auto elem = setting.value->listElems()[n];
-                    if (elem->normalType() != nString)
+                    if (elem->type() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
                     ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 93788273f..32c115c12 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->normalType() != nString) throw errMsg;
+        if ((*i)->type() != nString) throw errMsg;
         auto out = outputs.find((*i)->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
@@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames()
 bool DrvInfo::checkMeta(Value & v)
 {
     state->forceValue(v);
-    if (v.normalType() == nList) {
+    if (v.type() == nList) {
         for (unsigned int n = 0; n < v.listSize(); ++n)
             if (!checkMeta(*v.listElems()[n])) return false;
         return true;
     }
-    else if (v.normalType() == nAttrs) {
+    else if (v.type() == nAttrs) {
         Bindings::iterator i = v.attrs->find(state->sOutPath);
         if (i != v.attrs->end()) return false;
         for (auto & i : *v.attrs)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.normalType() == nInt || v.normalType() == nBool || v.normalType() == nString ||
-                v.normalType() == nFloat;
+    else return v.type() == nInt || v.type() == nBool || v.type() == nString ||
+                v.type() == nFloat;
 }
 
 
@@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name)
 string DrvInfo::queryMetaString(const string & name)
 {
     Value * v = queryMeta(name);
-    if (!v || v->normalType() != nString) return "";
+    if (!v || v->type() != nString) return "";
     return v->string.s;
 }
 
@@ -210,8 +210,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->normalType() == nInt) return v->integer;
-    if (v->normalType() == nString) {
+    if (v->type() == nInt) return v->integer;
+    if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
         NixInt n;
@@ -224,8 +224,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->normalType() == nFloat) return v->fpoint;
-    if (v->normalType() == nString) {
+    if (v->type() == nFloat) return v->fpoint;
+    if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
         NixFloat n;
@@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->normalType() == nBool) return v->boolean;
-    if (v->normalType() == nString) {
+    if (v->type() == nBool) return v->boolean;
+    if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
         if (strcmp(v->string.s, "true") == 0) return true;
@@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn,
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v.normalType() == nAttrs) {
+    else if (v.type() == nAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
@@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                 /* If the value of this attribute is itself a set,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
-                if (i->value->normalType() == nAttrs) {
+                if (i->value->type() == nAttrs) {
                     Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
@@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v.normalType() == nList) {
+    else if (v.type() == nList) {
         for (unsigned int n = 0; n < v.listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
             if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index f6ca612f4..4106f1ec8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -356,7 +356,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
 {
     state.forceValue(*args[0], pos);
     string t;
-    switch (args[0]->normalType()) {
+    switch (args[0]->type()) {
         case nInt: t = "int"; break;
         case nBool: t = "bool"; break;
         case nString: t = "string"; break;
@@ -389,7 +389,7 @@ static RegisterPrimOp primop_typeOf({
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nNull);
+    mkBool(v, args[0]->type() == nNull);
 }
 
 static RegisterPrimOp primop_isNull({
@@ -409,7 +409,7 @@ static RegisterPrimOp primop_isNull({
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nFunction);
+    mkBool(v, args[0]->type() == nFunction);
 }
 
 static RegisterPrimOp primop_isFunction({
@@ -425,7 +425,7 @@ static RegisterPrimOp primop_isFunction({
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nInt);
+    mkBool(v, args[0]->type() == nInt);
 }
 
 static RegisterPrimOp primop_isInt({
@@ -441,7 +441,7 @@ static RegisterPrimOp primop_isInt({
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nFloat);
+    mkBool(v, args[0]->type() == nFloat);
 }
 
 static RegisterPrimOp primop_isFloat({
@@ -457,7 +457,7 @@ static RegisterPrimOp primop_isFloat({
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nString);
+    mkBool(v, args[0]->type() == nString);
 }
 
 static RegisterPrimOp primop_isString({
@@ -473,7 +473,7 @@ static RegisterPrimOp primop_isString({
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nBool);
+    mkBool(v, args[0]->type() == nBool);
 }
 
 static RegisterPrimOp primop_isBool({
@@ -489,7 +489,7 @@ static RegisterPrimOp primop_isBool({
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nPath);
+    mkBool(v, args[0]->type() == nPath);
 }
 
 static RegisterPrimOp primop_isPath({
@@ -505,13 +505,13 @@ struct CompareValues
 {
     bool operator () (const Value * v1, const Value * v2) const
     {
-        if (v1->normalType() == nFloat && v2->normalType() == nInt)
+        if (v1->type() == nFloat && v2->type() == nInt)
             return v1->fpoint < v2->integer;
-        if (v1->normalType() == nInt && v2->normalType() == nFloat)
+        if (v1->type() == nInt && v2->type() == nFloat)
             return v1->integer < v2->fpoint;
-        if (v1->normalType() != v2->normalType())
+        if (v1->type() != v2->type())
             throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
-        switch (v1->normalType()) {
+        switch (v1->type()) {
             case nInt:
                 return v1->integer < v2->integer;
             case nFloat:
@@ -762,7 +762,7 @@ static RegisterPrimOp primop_deepSeq({
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    if (args[0]->normalType() == nString)
+    if (args[0]->type() == nString)
         printError("trace: %1%", args[0]->string.s);
     else
         printError("trace: %1%", *args[0]);
@@ -887,7 +887,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
             if (ignoreNulls) {
                 state.forceValue(*i->value, pos);
-                if (i->value->normalType() == nNull) continue;
+                if (i->value->type() == nNull) continue;
             }
 
             if (i->name == state.sContentAddressed) {
@@ -1293,7 +1293,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
-    if (args[0]->normalType() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
+    if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
 static RegisterPrimOp primop_dirOf({
@@ -1793,7 +1793,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         });
 
     state.forceValue(*args[0], pos);
-    if (args[0]->normalType() != nFunction)
+    if (args[0]->type() != nFunction)
         throw TypeError({
             .hint = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
@@ -2059,7 +2059,7 @@ static RegisterPrimOp primop_hasAttr({
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nAttrs);
+    mkBool(v, args[0]->type() == nAttrs);
 }
 
 static RegisterPrimOp primop_isAttrs({
@@ -2322,7 +2322,7 @@ static RegisterPrimOp primop_mapAttrs({
 static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->normalType() == nList);
+    mkBool(v, args[0]->type() == nList);
 }
 
 static RegisterPrimOp primop_isList({
@@ -2816,7 +2816,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat)
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos));
@@ -2835,7 +2835,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat)
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos));
@@ -2854,7 +2854,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat)
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos));
@@ -2881,7 +2881,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
             .errPos = pos
         });
 
-    if (args[0]->normalType() == nFloat || args[1]->normalType() == nFloat) {
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
     } else {
         NixInt i1 = state.forceInt(*args[0], pos);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 2461ebc99..845a1ed1b 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
 
     state.forceValue(*args[0]);
 
-    if (args[0]->normalType() == nAttrs) {
+    if (args[0]->type() == nAttrs) {
 
         state.forceAttrs(*args[0], pos);
 
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 6d93e1dc2..6e7ddde8e 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -85,25 +85,25 @@ static void fetchTree(
 
     state.forceValue(*args[0]);
 
-    if (args[0]->normalType() == nAttrs) {
+    if (args[0]->type() == nAttrs) {
         state.forceAttrs(*args[0], pos);
 
         fetchers::Attrs attrs;
 
         for (auto & attr : *args[0]->attrs) {
             state.forceValue(*attr.value);
-            if (attr.value->normalType() == nPath || attr.value->normalType() == nString)
+            if (attr.value->type() == nPath || attr.value->type() == nString)
                 addURI(
                     state,
                     attrs,
                     attr.name,
                     state.coerceToString(*attr.pos, *attr.value, context, false, false)
                 );
-            else if (attr.value->normalType() == nString)
+            else if (attr.value->type() == nString)
                 addURI(state, attrs, attr.name, attr.value->string.s);
-            else if (attr.value->normalType() == nBool)
+            else if (attr.value->type() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
-            else if (attr.value->normalType() == nInt)
+            else if (attr.value->type() == nInt)
                 attrs.emplace(attr.name, attr.value->integer);
             else
                 throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
@@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.forceValue(*args[0]);
 
-    if (args[0]->normalType() == nAttrs) {
+    if (args[0]->type() == nAttrs) {
 
         state.forceAttrs(*args[0], pos);
 
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index b5f4c8654..bfea24d40 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -16,7 +16,7 @@ void printValueAsJSON(EvalState & state, bool strict,
 
     if (strict) state.forceValue(v);
 
-    switch (v.normalType()) {
+    switch (v.type()) {
 
         case nInt:
             out.write(v.integer);
diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc
index 26be07cff..7464455d8 100644
--- a/src/libexpr/value-to-xml.cc
+++ b/src/libexpr/value-to-xml.cc
@@ -58,7 +58,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
     if (strict) state.forceValue(v);
 
-    switch (v.normalType()) {
+    switch (v.type()) {
 
         case nInt:
             doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
@@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->normalType() == nString)
+                    if (a->value->type() == nString)
                         xmlAttrs["drvPath"] = drvPath = a->value->string.s;
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->normalType() == nString)
+                    if (a->value->type() == nString)
                         xmlAttrs["outPath"] = a->value->string.s;
                 }
 
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 8b312bf03..61ea1d64b 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -136,12 +136,12 @@ public:
     // These should be removed eventually, by putting the functionality that's
     // needed by callers into methods of this type
 
-    // normalType() == nThunk
+    // type() == nThunk
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
     inline bool isBlackhole() const { return internalType == tBlackhole; };
 
-    // normalType() == nFunction
+    // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
     inline bool isPrimOp() const { return internalType == tPrimOp; };
     inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
@@ -204,7 +204,7 @@ public:
 
     // Returns the normal type of a Value. This only returns nThunk if the
     // Value hasn't been forceValue'd
-    inline ValueType normalType() const
+    inline ValueType type() const
     {
         switch (internalType) {
             case tInt: return nInt;
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 404fd5111..6c2e075ed 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                                         i.queryName(), j)
                                 });
                             else {
-                                if (v->normalType() == nString) {
+                                if (v->type() == nString) {
                                     attrs2["type"] = "string";
                                     attrs2["value"] = v->string.s;
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->normalType() == nInt) {
+                                } else if (v->type() == nInt) {
                                     attrs2["type"] = "int";
                                     attrs2["value"] = (format("%1%") % v->integer).str();
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->normalType() == nFloat) {
+                                } else if (v->type() == nFloat) {
                                     attrs2["type"] = "float";
                                     attrs2["value"] = (format("%1%") % v->fpoint).str();
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->normalType() == nBool) {
+                                } else if (v->type() == nBool) {
                                     attrs2["type"] = "bool";
                                     attrs2["value"] = v->boolean ? "true" : "false";
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->normalType() == nList) {
+                                } else if (v->type() == nList) {
                                     attrs2["type"] = "strings";
                                     XMLOpenElement m(xml, "meta", attrs2);
                                     for (unsigned int j = 0; j < v->listSize(); ++j) {
-                                        if (v->listElems()[j]->normalType() != nString) continue;
+                                        if (v->listElems()[j]->type() != nString) continue;
                                         XMLAttrs attrs3;
                                         attrs3["value"] = v->listElems()[j]->string.s;
                                         xml.writeEmptyElement("string", attrs3);
                                     }
-                              } else if (v->normalType() == nAttrs) {
+                              } else if (v->type() == nAttrs) {
                                   attrs2["type"] = "strings";
                                   XMLOpenElement m(xml, "meta", attrs2);
                                   Bindings & attrs = *v->attrs;
                                   for (auto &i : attrs) {
                                       Attr & a(*attrs.find(i.name));
-                                      if(a.value->normalType() != nString) continue;
+                                      if(a.value->type() != nString) continue;
                                       XMLAttrs attrs3;
                                       attrs3["type"] = i.name;
                                       attrs3["value"] = a.value->string.s;
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index bba3b1bc6..ea82e5300 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -97,10 +97,10 @@ struct CmdEval : MixJSON, InstallableCommand
             recurse = [&](Value & v, const Pos & pos, const Path & path)
             {
                 state->forceValue(v);
-                if (v.normalType() == nString)
+                if (v.type() == nString)
                     // FIXME: disallow strings with contexts?
                     writeFile(path, v.string.s);
-                else if (v.normalType() == nAttrs) {
+                else if (v.type() == nAttrs) {
                     if (mkdir(path.c_str(), 0777) == -1)
                         throw SysError("creating directory '%s'", path);
                     for (auto & attr : *v.attrs)
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index e4da0348c..066430c5d 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -279,7 +279,7 @@ struct CmdFlakeCheck : FlakeCommand
                 if (v.isLambda()) {
                     if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
                         throw Error("module must match an open attribute set ('{ config, ... }')");
-                } else if (v.normalType() == nAttrs) {
+                } else if (v.type() == nAttrs) {
                     for (auto & attr : *v.attrs)
                         try {
                             state->forceValue(*attr.value, *attr.pos);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 047e2dc59..673155078 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -446,7 +446,7 @@ bool NixRepl::processLine(string line)
 
         Pos pos;
 
-        if (v.normalType() == nPath || v.normalType() == nString) {
+        if (v.type() == nPath || v.type() == nString) {
             PathSet context;
             auto filename = state->coerceToString(noPos, v, context);
             pos.file = state->symbols.create(filename);
@@ -669,7 +669,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 
     state->forceValue(v);
 
-    switch (v.normalType()) {
+    switch (v.type()) {
 
     case nInt:
         str << ANSI_CYAN << v.integer << ANSI_NORMAL;

From 7de4b1e9aa72c21f5c98001ea2a5f312d26634b7 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Thu, 17 Dec 2020 23:42:49 +0100
Subject: [PATCH 581/882] smaller fixes

---
 doc/manual/src/SUMMARY.md.in           |  1 +
 doc/manual/src/command-ref/nix.md      |  1 -
 doc/manual/src/contributing/hacking.md | 76 +++++++++++++++++++++++++
 doc/manual/src/hacking.md              | 77 --------------------------
 4 files changed, 77 insertions(+), 78 deletions(-)
 delete mode 100644 doc/manual/src/command-ref/nix.md
 delete mode 100644 doc/manual/src/hacking.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index fdb1f7969..448fee803 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -62,6 +62,7 @@
     - [nix-instantiate](command-ref/nix-instantiate.md)
     - [nix-prefetch-url](command-ref/nix-prefetch-url.md)
   - [Experimental Commands](command-ref/experimental-commands.md)
+@manpages@
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
 - [Glossary](glossary.md)
diff --git a/doc/manual/src/command-ref/nix.md b/doc/manual/src/command-ref/nix.md
deleted file mode 100644
index cc677d127..000000000
--- a/doc/manual/src/command-ref/nix.md
+++ /dev/null
@@ -1 +0,0 @@
-# nix
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 2ad773dea..2a1e55e5b 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -1 +1,77 @@
 # Hacking
+
+This section provides some notes on how to hack on Nix. To get the
+latest version of Nix from GitHub:
+
+```console
+$ git clone https://github.com/NixOS/nix.git
+$ cd nix
+```
+
+To build Nix for the current operating system/architecture use
+
+```console
+$ nix-build
+```
+
+or if you have a flake-enabled nix:
+
+```console
+$ nix build
+```
+
+This will build `defaultPackage` attribute defined in the `flake.nix`
+file. To build for other platforms add one of the following suffixes to
+it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e.
+
+```console
+$ nix-build -A defaultPackage.x86_64-linux
+```
+
+To build all dependencies and start a shell in which all environment
+variables are set up so that those dependencies can be found:
+
+```console
+$ nix-shell
+```
+
+To build Nix itself in this shell:
+
+```console
+[nix-shell]$ ./bootstrap.sh
+[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out
+[nix-shell]$ make -j $NIX_BUILD_CORES
+```
+
+To install it in `$(pwd)/outputs` and test it:
+
+```console
+[nix-shell]$ make install
+[nix-shell]$ make installcheck -j $NIX_BUILD_CORES
+[nix-shell]$ ./outputs/out/bin/nix --version
+nix (Nix) 3.0
+```
+
+To run a functional test:
+
+```console
+make tests/test-name-should-auto-complete.sh.test
+```
+
+To run the unit-tests for C++ code:
+
+```
+make check
+```
+
+If you have a flakes-enabled Nix you can replace:
+
+```console
+$ nix-shell
+```
+
+by:
+
+```console
+$ nix develop
+```
diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
deleted file mode 100644
index 2a1e55e5b..000000000
--- a/doc/manual/src/hacking.md
+++ /dev/null
@@ -1,77 +0,0 @@
-# Hacking
-
-This section provides some notes on how to hack on Nix. To get the
-latest version of Nix from GitHub:
-
-```console
-$ git clone https://github.com/NixOS/nix.git
-$ cd nix
-```
-
-To build Nix for the current operating system/architecture use
-
-```console
-$ nix-build
-```
-
-or if you have a flake-enabled nix:
-
-```console
-$ nix build
-```
-
-This will build `defaultPackage` attribute defined in the `flake.nix`
-file. To build for other platforms add one of the following suffixes to
-it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e.
-
-```console
-$ nix-build -A defaultPackage.x86_64-linux
-```
-
-To build all dependencies and start a shell in which all environment
-variables are set up so that those dependencies can be found:
-
-```console
-$ nix-shell
-```
-
-To build Nix itself in this shell:
-
-```console
-[nix-shell]$ ./bootstrap.sh
-[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out
-[nix-shell]$ make -j $NIX_BUILD_CORES
-```
-
-To install it in `$(pwd)/outputs` and test it:
-
-```console
-[nix-shell]$ make install
-[nix-shell]$ make installcheck -j $NIX_BUILD_CORES
-[nix-shell]$ ./outputs/out/bin/nix --version
-nix (Nix) 3.0
-```
-
-To run a functional test:
-
-```console
-make tests/test-name-should-auto-complete.sh.test
-```
-
-To run the unit-tests for C++ code:
-
-```
-make check
-```
-
-If you have a flakes-enabled Nix you can replace:
-
-```console
-$ nix-shell
-```
-
-by:
-
-```console
-$ nix develop
-```

From b70d22baca3e8826392b61aa53955c6da74b8724 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Fri, 18 Dec 2020 14:38:49 +0100
Subject: [PATCH 582/882] Replace Value type setters with mk* functions

Move clearValue inside Value

mkInt instead of setInt

mkBool instead of setBool

mkString instead of setString

mkPath instead of setPath

mkNull instead of setNull

mkAttrs instead of setAttrs

mkList instead of setList*

mkThunk instead of setThunk

mkApp instead of setApp

mkLambda instead of setLambda

mkBlackhole instead of setBlackhole

mkPrimOp instead of setPrimOp

mkPrimOpApp instead of setPrimOpApp

mkExternal instead of setExternal

mkFloat instead of setFloat

Add note that the static mk* function should be removed eventually
---
 src/libexpr/attr-set.cc    |   4 +-
 src/libexpr/eval-inline.hh |   6 +-
 src/libexpr/eval.cc        |  49 +++-------
 src/libexpr/nixexpr.hh     |   2 +-
 src/libexpr/primops.cc     |   2 +-
 src/libexpr/value.hh       | 193 ++++++++++++++++++++++++-------------
 src/nix/repl.cc            |   4 +-
 7 files changed, 144 insertions(+), 116 deletions(-)

diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index 17886a426..b6091c955 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
         v = vEmptySet;
         return;
     }
-    clearValue(v);
-    v.setAttrs();
-    v.attrs = allocBindings(capacity);
+    v.mkAttrs(allocBindings(capacity));
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
 }
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index e56ce261c..f6dead6b0 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -36,13 +36,11 @@ void EvalState::forceValue(Value & v, const Pos & pos)
         Env * env = v.thunk.env;
         Expr * expr = v.thunk.expr;
         try {
-            v.setBlackhole();
+            v.mkBlackhole();
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
-            v.setThunk();
-            v.thunk.env = env;
-            v.thunk.expr = expr;
+            v.mkThunk(env, expr);
             throw;
         }
     }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2f8d6d259..5a641d02c 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -430,9 +430,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         }
     }
 
-    clearValue(vEmptySet);
-    vEmptySet.setAttrs();
-    vEmptySet.attrs = allocBindings(0);
+    vEmptySet.mkAttrs(allocBindings(0));
 
     createBaseEnv();
 }
@@ -548,16 +546,14 @@ Value * EvalState::addPrimOp(const string & name,
        the primop to a dummy value. */
     if (arity == 0) {
         auto vPrimOp = allocValue();
-        vPrimOp->setPrimOp();
-        vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
+        vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym });
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
     }
 
     Value * v = allocValue();
-    v->setPrimOp();
-    v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
+    v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -572,8 +568,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     if (primOp.arity == 0) {
         primOp.arity = 1;
         auto vPrimOp = allocValue();
-        vPrimOp->setPrimOp();
-        vPrimOp->primOp = new PrimOp(std::move(primOp));
+        vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(primOp.name, v);
@@ -584,8 +579,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         primOp.name = symbols.create(std::string(primOp.name, 2));
 
     Value * v = allocValue();
-    v->setPrimOp();
-    v->primOp = new PrimOp(std::move(primOp));
+    v->mkPrimOp(new PrimOp(std::move(primOp)));
     staticBaseEnv.vars[envName] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
@@ -708,15 +702,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con
 
 void mkString(Value & v, const char * s)
 {
-    mkStringNoCopy(v, dupString(s));
+    v.mkString(dupString(s));
 }
 
 
 Value & mkString(Value & v, std::string_view s, const PathSet & context)
 {
-    v.setString();
-    v.string.s = dupStringWithLen(s.data(), s.size());
-    v.string.context = 0;
+    v.mkString(dupStringWithLen(s.data(), s.size()));
     if (!context.empty()) {
         size_t n = 0;
         v.string.context = (const char * *)
@@ -731,7 +723,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context)
 
 void mkPath(Value & v, const char * s)
 {
-    mkPathNoCopy(v, dupString(s));
+    v.mkPath(dupString(s));
 }
 
 
@@ -792,16 +784,9 @@ Env & EvalState::allocEnv(size_t size)
 
 void EvalState::mkList(Value & v, size_t size)
 {
-    clearValue(v);
-    if (size == 1)
-        v.setList1();
-    else if (size == 2)
-        v.setList2();
-    else {
-        v.setListN();
-        v.bigList.size = size;
-        v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0;
-    }
+    v.mkList(size);
+    if (size > 2)
+        v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *));
     nrListElems += size;
 }
 
@@ -810,9 +795,7 @@ unsigned long nrThunks = 0;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
-    v.setThunk();
-    v.thunk.env = &env;
-    v.thunk.expr = expr;
+    v.mkThunk(&env, expr);
     nrThunks++;
 }
 
@@ -1207,9 +1190,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
 
 void ExprLambda::eval(EvalState & state, Env & env, Value & v)
 {
-    v.setLambda();
-    v.lambda.env = &env;
-    v.lambda.fun = this;
+    v.mkLambda(&env, this);
 }
 
 
@@ -1252,9 +1233,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     } else {
         Value * fun2 = allocValue();
         *fun2 = fun;
-        v.setPrimOpApp();
-        v.primOpApp.left = fun2;
-        v.primOpApp.right = &arg;
+        v.mkPrimOpApp(fun2, &arg);
     }
 }
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index e4cbc660f..b80a7de4e 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -129,7 +129,7 @@ struct ExprPath : Expr
 {
     string s;
     Value v;
-    ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
+    ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4106f1ec8..45066e9cf 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1434,7 +1434,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
         Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
         if (ent.type == DT_UNKNOWN)
             ent.type = getFileType(path + "/" + ent.name);
-        mkStringNoCopy(*ent_val,
+        ent_val->mkString(
             ent.type == DT_REG ? "regular" :
             ent.type == DT_DIR ? "directory" :
             ent.type == DT_LNK ? "symlink" :
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 61ea1d64b..b317c1898 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -114,24 +114,6 @@ friend void printValue(std::ostream & str, std::set<const Value *> & active, con
 
 public:
 
-    inline void setInt() { internalType = tInt; };
-    inline void setBool() { internalType = tBool; };
-    inline void setString() { internalType = tString; };
-    inline void setPath() { internalType = tPath; };
-    inline void setNull() { internalType = tNull; };
-    inline void setAttrs() { internalType = tAttrs; };
-    inline void setList1() { internalType = tList1; };
-    inline void setList2() { internalType = tList2; };
-    inline void setListN() { internalType = tListN; };
-    inline void setThunk() { internalType = tThunk; };
-    inline void setApp() { internalType = tApp; };
-    inline void setLambda() { internalType = tLambda; };
-    inline void setBlackhole() { internalType = tBlackhole; };
-    inline void setPrimOp() { internalType = tPrimOp; };
-    inline void setPrimOpApp() { internalType = tPrimOpApp; };
-    inline void setExternal() { internalType = tExternal; };
-    inline void setFloat() { internalType = tFloat; };
-
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
     // needed by callers into methods of this type
@@ -222,6 +204,123 @@ public:
         abort();
     }
 
+    /* After overwriting an app node, be sure to clear pointers in the
+       Value to ensure that the target isn't kept alive unnecessarily. */
+    inline void clearValue()
+    {
+        app.left = app.right = 0;
+    }
+
+    inline void mkInt(NixInt n)
+    {
+        clearValue();
+        internalType = tInt;
+        integer = n;
+    }
+
+    inline void mkBool(bool b)
+    {
+        clearValue();
+        internalType = tBool;
+        boolean = b;
+    }
+
+    inline void mkString(const char * s, const char * * context = 0)
+    {
+        internalType = tString;
+        string.s = s;
+        string.context = context;
+    }
+
+    inline void mkPath(const char * s)
+    {
+        clearValue();
+        internalType = tPath;
+        path = s;
+    }
+
+    inline void mkNull()
+    {
+        clearValue();
+        internalType = tNull;
+    }
+
+    inline void mkAttrs(Bindings * a)
+    {
+        clearValue();
+        internalType = tAttrs;
+        attrs = a;
+    }
+
+    inline void mkList(size_t size)
+    {
+        clearValue();
+        if (size == 1)
+            internalType = tList1;
+        else if (size == 2)
+            internalType = tList2;
+        else {
+            internalType = tListN;
+            bigList.size = size;
+        }
+    }
+
+    inline void mkThunk(Env * e, Expr * ex)
+    {
+        internalType = tThunk;
+        thunk.env = e;
+        thunk.expr = ex;
+    }
+
+    inline void mkApp(Value * l, Value * r)
+    {
+        internalType = tApp;
+        app.left = l;
+        app.right = r;
+    }
+
+    inline void mkLambda(Env * e, ExprLambda * f)
+    {
+        internalType = tLambda;
+        lambda.env = e;
+        lambda.fun = f;
+    }
+
+    inline void mkBlackhole()
+    {
+        internalType = tBlackhole;
+        // Value will be overridden anyways
+    }
+
+    inline void mkPrimOp(PrimOp * p)
+    {
+        clearValue();
+        internalType = tPrimOp;
+        primOp = p;
+    }
+
+
+    inline void mkPrimOpApp(Value * l, Value * r)
+    {
+        internalType = tPrimOpApp;
+        app.left = l;
+        app.right = r;
+    }
+
+    inline void mkExternal(ExternalValueBase * e)
+    {
+        clearValue();
+        internalType = tExternal;
+        external = e;
+    }
+
+    inline void mkFloat(NixFloat n)
+    {
+        clearValue();
+        internalType = tFloat;
+        fpoint = n;
+    }
+
     bool isList() const
     {
         return internalType == tList1 || internalType == tList2 || internalType == tListN;
@@ -251,86 +350,42 @@ public:
 };
 
 
-/* After overwriting an app node, be sure to clear pointers in the
-   Value to ensure that the target isn't kept alive unnecessarily. */
-static inline void clearValue(Value & v)
-{
-    v.app.left = v.app.right = 0;
-}
-
 
+// TODO: Remove these static functions, replace call sites with v.mk* instead
 static inline void mkInt(Value & v, NixInt n)
 {
-    clearValue(v);
-    v.setInt();
-    v.integer = n;
+    v.mkInt(n);
 }
 
-
 static inline void mkFloat(Value & v, NixFloat n)
 {
-    clearValue(v);
-    v.setFloat();
-    v.fpoint = n;
+    v.mkFloat(n);
 }
 
-
 static inline void mkBool(Value & v, bool b)
 {
-    clearValue(v);
-    v.setBool();
-    v.boolean = b;
+    v.mkBool(b);
 }
 
-
 static inline void mkNull(Value & v)
 {
-    clearValue(v);
-    v.setNull();
+    v.mkNull();
 }
 
-
 static inline void mkApp(Value & v, Value & left, Value & right)
 {
-    v.setApp();
-    v.app.left = &left;
-    v.app.right = &right;
+    v.mkApp(&left, &right);
 }
 
-
-static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
-{
-    v.setPrimOpApp();
-    v.app.left = &left;
-    v.app.right = &right;
-}
-
-
-static inline void mkStringNoCopy(Value & v, const char * s)
-{
-    v.setString();
-    v.string.s = s;
-    v.string.context = 0;
-}
-
-
 static inline void mkString(Value & v, const Symbol & s)
 {
-    mkStringNoCopy(v, ((const string &) s).c_str());
+    v.mkString(((const string &) s).c_str());
 }
 
 
 void mkString(Value & v, const char * s);
 
 
-static inline void mkPathNoCopy(Value & v, const char * s)
-{
-    clearValue(v);
-    v.setPath();
-    v.path = s;
-}
-
-
 void mkPath(Value & v, const char * s);
 
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 673155078..a992d8732 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -551,9 +551,7 @@ bool NixRepl::processLine(string line)
         {
             Expr * e = parseString(string(line, p + 1));
             Value & v(*state->allocValue());
-            v.setThunk();
-            v.thunk.env = env;
-            v.thunk.expr = e;
+            v.mkThunk(env, e);
             addVarToScope(state->symbols.create(name), v);
         } else {
             Value v;

From 1a1af75338cb9ed28dc00de2e696d8efc5d37287 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 15:33:12 +0000
Subject: [PATCH 583/882] Overhaul store subclassing

We embrace virtual the rest of the way, and get rid of the
`assert(false)` 0-param constructors.

We also list config base classes first, so the constructor order is
always:

  1. all the configs
  2. all the stores

Each in the same order
---
 src/libstore/binary-cache-store.hh       |  2 +-
 src/libstore/build/derivation-goal.cc    | 11 ++++++++---
 src/libstore/dummy-store.cc              |  3 ++-
 src/libstore/http-binary-cache-store.cc  |  5 ++++-
 src/libstore/legacy-ssh-store.cc         |  3 ++-
 src/libstore/local-binary-cache-store.cc |  5 ++++-
 src/libstore/local-fs-store.hh           |  2 +-
 src/libstore/local-store.cc              |  2 ++
 src/libstore/local-store.hh              |  2 +-
 src/libstore/remote-store.cc             |  4 ++--
 src/libstore/remote-store.hh             |  2 +-
 src/libstore/s3-binary-cache-store.cc    | 11 ++++++++++-
 src/libstore/s3-binary-cache-store.hh    |  6 ++----
 src/libstore/ssh-store.cc                |  4 +++-
 src/libstore/store-api.hh                | 20 +-------------------
 src/libstore/uds-remote-store.cc         |  3 +++
 src/libstore/uds-remote-store.hh         |  7 +------
 17 files changed, 48 insertions(+), 44 deletions(-)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 07a8b2beb..443a53cac 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
         "enable multi-threading compression, available for xz only currently"};
 };
 
-class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
+class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
 {
 
 private:
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index f494545fb..47d11dc53 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1985,7 +1985,7 @@ void DerivationGoal::writeStructuredAttrs()
     chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
-struct RestrictedStoreConfig : LocalFSStoreConfig
+struct RestrictedStoreConfig : virtual LocalFSStoreConfig
 {
     using LocalFSStoreConfig::LocalFSStoreConfig;
     const std::string name() { return "Restricted Store"; }
@@ -1994,14 +1994,19 @@ struct RestrictedStoreConfig : LocalFSStoreConfig
 /* A wrapper around LocalStore that only allows building/querying of
    paths that are in the input closures of the build or were added via
    recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
+struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
 {
     ref<LocalStore> next;
 
     DerivationGoal & goal;
 
     RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , RestrictedStoreConfig(params)
+        , Store(params)
+        , LocalFSStore(params)
+        , next(next), goal(goal)
     { }
 
     Path getRealStoreDir() override
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 91fc178db..3c7caf8f2 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig {
     const std::string name() override { return "Dummy Store"; }
 };
 
-struct DummyStore : public Store, public virtual DummyStoreConfig
+struct DummyStore : public virtual DummyStoreConfig, public virtual Store
 {
     DummyStore(const std::string scheme, const std::string uri, const Params & params)
         : DummyStore(params)
@@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
 
     DummyStore(const Params & params)
         : StoreConfig(params)
+        , DummyStoreConfig(params)
         , Store(params)
     { }
 
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 9d2a89f96..0a3afcd51 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     const std::string name() override { return "Http Binary Cache Store"; }
 };
 
-class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
+class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore
 {
 private:
 
@@ -36,6 +36,9 @@ public:
         const Path & _cacheUri,
         const Params & params)
         : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , HttpBinaryCacheStoreConfig(params)
+        , Store(params)
         , BinaryCacheStore(params)
         , cacheUri(scheme + "://" + _cacheUri)
     {
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index ad1779aea..253c0033e 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -22,7 +22,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
     const std::string name() override { return "Legacy SSH Store"; }
 };
 
-struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
+struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
 {
     // Hack for getting remote build log output.
     // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
@@ -48,6 +48,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
 
     LegacySSHStore(const string & scheme, const string & host, const Params & params)
         : StoreConfig(params)
+        , LegacySSHStoreConfig(params)
         , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index bb7464989..a58b7733f 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -11,7 +11,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     const std::string name() override { return "Local Binary Cache Store"; }
 };
 
-class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
+class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore
 {
 private:
 
@@ -24,6 +24,9 @@ public:
         const Path & binaryCacheDir,
         const Params & params)
         : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , LocalBinaryCacheStoreConfig(params)
+        , Store(params)
         , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index 8eccd8236..55941b771 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig
         "log", "directory where Nix will store state"};
 };
 
-class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
+class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
 {
 public:
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e9f9bde4d..c52d4b62a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -100,6 +100,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 
 LocalStore::LocalStore(const Params & params)
     : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , LocalStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
     , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 877dba742..ae9497b2e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 };
 
 
-class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
+class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore
 {
 private:
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f1f4d0516..be07f02dc 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -77,8 +77,8 @@ void write(const Store & store, Sink & out, const std::optional<ContentAddress>
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
-    : Store(params)
-    , RemoteStoreConfig(params)
+    : RemoteStoreConfig(params)
+    , Store(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
             [this]() {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index fdd53e6ed..b3a9910a3 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
+class RemoteStore : public virtual RemoteStoreConfig, public virtual Store
 {
 public:
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index d6edafd7e..6bfbee044 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -177,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject(
     return res;
 }
 
+S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
+    : BinaryCacheStoreConfig(params)
+    , BinaryCacheStore(params)
+{ }
+
 struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
@@ -195,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     const std::string name() override { return "S3 Binary Cache Store"; }
 };
 
-struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
+struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
 {
     std::string bucketName;
 
@@ -208,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
         const std::string & bucketName,
         const Params & params)
         : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , S3BinaryCacheStoreConfig(params)
+        , Store(params)
+        , BinaryCacheStore(params)
         , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh
index 4d43fe4d2..bce828b11 100644
--- a/src/libstore/s3-binary-cache-store.hh
+++ b/src/libstore/s3-binary-cache-store.hh
@@ -6,13 +6,11 @@
 
 namespace nix {
 
-class S3BinaryCacheStore : public BinaryCacheStore
+class S3BinaryCacheStore : public virtual BinaryCacheStore
 {
 protected:
 
-    S3BinaryCacheStore(const Params & params)
-        : BinaryCacheStore(params)
-    { }
+    S3BinaryCacheStore(const Params & params);
 
 public:
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 08d0bd565..17c258201 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -20,12 +20,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
     const std::string name() override { return "SSH Store"; }
 };
 
-class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
+class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
 {
 public:
 
     SSHStore(const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , SSHStoreConfig(params)
         , Store(params)
         , RemoteStore(params)
         , host(host)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ce95b78b1..9bcff08eb 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -175,25 +175,7 @@ struct StoreConfig : public Config
 {
     using Config::Config;
 
-    /**
-     * When constructing a store implementation, we pass in a map `params` of
-     * parameters that's supposed to initialize the associated config.
-     * To do that, we must use the `StoreConfig(StringMap & params)`
-     * constructor, so we'd like to `delete` its default constructor to enforce
-     * it.
-     *
-     * However, actually deleting it means that all the subclasses of
-     * `StoreConfig` will have their default constructor deleted (because it's
-     * supposed to call the deleted default constructor of `StoreConfig`). But
-     * because we're always using virtual inheritance, the constructors of
-     * child classes will never implicitely call this one, so deleting it will
-     * be more painful than anything else.
-     *
-     * So we `assert(false)` here to ensure at runtime that the right
-     * constructor is always called without having to redefine a custom
-     * constructor for each `*Config` class.
-     */
-    StoreConfig() { assert(false); }
+    StoreConfig() = delete;
 
     virtual ~StoreConfig() { }
 
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index 24f3e9c6d..cac4fa036 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -15,6 +15,9 @@ namespace nix {
 
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , RemoteStoreConfig(params)
+    , UDSRemoteStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
     , RemoteStore(params)
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
index e5de104c9..ddc7716cd 100644
--- a/src/libstore/uds-remote-store.hh
+++ b/src/libstore/uds-remote-store.hh
@@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
     {
     }
 
-    UDSRemoteStoreConfig()
-        : UDSRemoteStoreConfig(Store::Params({}))
-    {
-    }
-
     const std::string name() override { return "Local Daemon Store"; }
 };
 
-class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
+class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore
 {
 public:
 

From 346baec783a7423aa5b6cacaf7eebb8d22d4ce79 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 7 Dec 2020 13:04:24 +0100
Subject: [PATCH 584/882] Move doc() to Args

---
 src/libutil/args.cc | 4 ++--
 src/libutil/args.hh | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 61f9503ec..a929ea5ac 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -254,6 +254,8 @@ nlohmann::json Args::toJSON()
     res["description"] = description();
     res["flags"] = std::move(flags);
     res["args"] = std::move(args);
+    auto s = doc();
+    if (s != "") res.emplace("doc", stripIndentation(s));
     return res;
 }
 
@@ -378,8 +380,6 @@ nlohmann::json Command::toJSON()
 
     auto res = Args::toJSON();
     res["examples"] = std::move(exs);
-    auto s = doc();
-    if (s != "") res.emplace("doc", stripIndentation(s));
     return res;
 }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 8069fd70f..68bbbb4f7 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -25,6 +25,9 @@ public:
     /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
+    /* Return documentation about this command, in Markdown format. */
+    virtual std::string doc() { return ""; }
+
 protected:
 
     static const size_t ArityAny = std::numeric_limits<size_t>::max();
@@ -225,9 +228,6 @@ struct Command : virtual Args
     virtual void prepare() { };
     virtual void run() = 0;
 
-    /* Return documentation about this command, in Markdown format. */
-    virtual std::string doc() { return ""; }
-
     struct Example
     {
         std::string description;

From ae7351dbeed3d03087b6b56d23b4b3942de1507d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 14:19:36 +0100
Subject: [PATCH 585/882] Add 'nix build' manpage

---
 src/nix/build.cc | 19 +++-------
 src/nix/build.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+), 15 deletions(-)
 create mode 100644 src/nix/build.md

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 67be4024b..c2974d983 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -43,22 +43,11 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
         return "build a derivation or fetch a store path";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To build and run GNU Hello from NixOS 17.03:",
-                "nix build -f channel:nixos-17.03 hello; ./result/bin/hello"
-            },
-            Example{
-                "To build the build.x86_64-linux attribute from release.nix:",
-                "nix build -f release.nix build.x86_64-linux"
-            },
-            Example{
-                "To make a profile point at GNU Hello:",
-                "nix build --profile /tmp/profile nixpkgs#hello"
-            },
-        };
+        return
+          #include "build.md"
+          ;
     }
 
     void run(ref<Store> store) override
diff --git a/src/nix/build.md b/src/nix/build.md
new file mode 100644
index 000000000..c2f3e387a
--- /dev/null
+++ b/src/nix/build.md
@@ -0,0 +1,92 @@
+R""(
+
+# Examples
+
+* Build the default package from the flake in the current directory:
+
+  ```console
+  # nix build
+  ```
+
+* Build and run GNU Hello from the `nixpkgs` flake:
+
+  ```console
+  # nix build nixpkgs#hello
+  # ./result/bin/hello
+  Hello, world!
+  ```
+
+* Build GNU Hello and Cowsay, leaving two result symlinks:
+
+  ```console
+  # nix build nixpkgs#hello nixpkgs#cowsay
+  # ls -l result*
+  lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+  lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2
+  ```
+
+* Build a specific output:
+
+  ```console
+  # nix build nixpkgs#glibc.dev
+  # ls -ld ./result-dev
+  lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev
+  ```
+
+* Build attribute `build.x86_64-linux` from (non-flake) Nix expression
+  `release.nix`:
+
+  ```console
+  # nix build -f release.nix build.x86_64-linux
+  ```
+
+* Build a NixOS system configuration from a flake, and make a profile
+  point to the result:
+
+  ```console
+  # nix build --profile /nix/var/nix/profiles/system \
+      ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel
+  ```
+
+  (This is essentially what `nixos-rebuild` does.)
+
+* Build an expression specified on the command line:
+
+  ```console
+  # nix build --impure --expr \
+      'with import <nixpkgs> {};
+       runCommand "foo" {
+         buildInputs = [ hello ];
+       }
+       "hello > $out"'
+  # cat ./result
+  Hello, world!
+  ```
+
+  Note that `--impure` is needed because we're using `<nixpkgs>`,
+  which relies on the `$NIX_PATH` environment variable.
+
+* Fetch a store path from the configured substituters, if it doesn't
+  already exist:
+
+  ```console
+  # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2
+  ```
+
+# Description
+
+`nix build` builds the specified *installables*. Installables that
+resolve to derivations are built (or substituted if possible). Store
+path installables are substituted.
+
+Unless `--no-link` is specified, after a successful build, it creates
+symlinks to the store paths of the installables. These symlinks have
+the prefix `./result` by default; this can be overriden using the
+`--out-link` option. Each symlink has a suffix `-<N>-<outname>`, where
+*N* is the index of the installable (with the left-most installable
+having index 0), and *outname* is the symbolic derivation output name
+(e.g. `bin`, `dev` or `lib`). `-<N>` is omitted if *N* = 0, and
+`-<outname>` is omitted if *outname* = `out` (denoting the default
+output).
+
+)""

From 09660b855778531be14968b720308d092af4dd2e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 17:16:23 +0100
Subject: [PATCH 586/882] Add 'nix run' and 'nix shell' manpages

---
 src/nix/run.cc   | 42 +++++------------------
 src/nix/run.md   | 87 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/nix/shell.md | 48 ++++++++++++++++++++++++++
 3 files changed, 143 insertions(+), 34 deletions(-)
 create mode 100644 src/nix/run.md
 create mode 100644 src/nix/shell.md

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 92a52c6cd..c2400f9ee 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -86,26 +86,11 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
         return "run a shell in which the specified packages are available";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To start a shell providing GNU Hello from NixOS 20.03:",
-                "nix shell nixpkgs/nixos-20.03#hello"
-            },
-            Example{
-                "To start a shell providing youtube-dl from your 'nixpkgs' channel:",
-                "nix shell nixpkgs#youtube-dl"
-            },
-            Example{
-                "To run GNU Hello:",
-                "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'"
-            },
-            Example{
-                "To run GNU Hello in a chroot store:",
-                "nix shell --store ~/my-nix nixpkgs#hello -c hello"
-            },
-        };
+        return
+          #include "shell.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -168,22 +153,11 @@ struct CmdRun : InstallableCommand, RunCommon
         return "run a Nix application";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To run Blender:",
-                "nix run blender-bin"
-            },
-            Example{
-                "To run vim from nixpkgs:",
-                "nix run nixpkgs#vim"
-            },
-            Example{
-                "To run vim from nixpkgs with arguments:",
-                "nix run nixpkgs#vim -- --help"
-            },
-        };
+        return
+          #include "run.md"
+          ;
     }
 
     Strings getDefaultFlakeAttrPaths() override
diff --git a/src/nix/run.md b/src/nix/run.md
new file mode 100644
index 000000000..c178e8b13
--- /dev/null
+++ b/src/nix/run.md
@@ -0,0 +1,87 @@
+R""(
+
+# Examples
+
+* Run the default app from the `blender-bin` flake:
+
+  ```console
+  # nix run blender-bin
+  ```
+
+* Run a non-default app from the `blender-bin` flake:
+
+  ```console
+  # nix run blender-bin#blender_2_83
+  ```
+
+  Tip: you can find apps provided by this flake by running `nix flake
+  show blender-bin`.
+
+* Run `vim` from the `nixpkgs` flake:
+
+  ```console
+  # nix run nixpkgs#vim
+  ```
+
+  Note that `vim` (as of the time of writing of this page) is not an
+  app but a package. Thus, Nix runs the eponymous file from the `vim`
+  package.
+
+* Run `vim` with arguments:
+
+  ```console
+  # nix run nixpkgs#vim -- --help
+  ```
+
+# Description
+
+`nix run` builds and runs *installable*, which must evaluate to an
+*app* or a regular Nix derivation.
+
+If *installable* evaluates to an *app* (see below), it executes the
+program specified by the app definition.
+
+If *installable* evaluates to a derivation, it will try to execute the
+program `<out>/bin/<name>`, where *out* is the primary output store
+path of the derivation and *name* is the name part of the value of the
+`name` attribute of the derivation (e.g. if `name` is set to
+`hello-1.10`, it will run `$out/bin/hello`).
+
+# Flake output attributes
+
+If no flake output attribute is given, `nix run` tries the following
+flake output attributes:
+
+* `defaultApp.<system>`
+
+* `defaultPackage.<system>`
+
+If an attribute *name* is given, `nix run` tries the following flake
+output attributes:
+
+* `apps.<system>.<name>`
+
+* `packages.<system>.<name>`
+
+* `legacyPackages.<system>.<name>`
+
+# Apps
+
+An app is specified by a flake output attribute named
+`apps.<system>.<name>` or `defaultApp.<system>`. It looks like this:
+
+```nix
+apps.x86_64-linux.blender_2_79 = {
+  type = "app";
+  program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender";
+};
+```
+
+The only supported attributes are:
+
+* `type` (required): Must be set to `app`.
+
+* `program` (required): The full path of the executable to run. It
+  must reside in the Nix store.
+
+)""
diff --git a/src/nix/shell.md b/src/nix/shell.md
new file mode 100644
index 000000000..2a379e03f
--- /dev/null
+++ b/src/nix/shell.md
@@ -0,0 +1,48 @@
+R""(
+
+# Examples
+
+* Start a shell providing `youtube-dl` from the `nixpkgs` flake:
+
+  ```console
+  # nix shell nixpkgs#youtube-dl
+  # youtube-dl --version
+  2020.11.01.1
+  ```
+
+* Start a shell providing GNU Hello from NixOS 20.03:
+
+  ```console
+  # nix shell nixpkgs/nixos-20.03#hello
+  ```
+
+* Run GNU Hello:
+
+  ```console
+  # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'
+  Hi everybody!
+  ```
+
+* Run GNU Hello in a chroot store:
+
+  ```console
+  # nix shell --store ~/my-nix nixpkgs#hello -c hello
+  ```
+
+* Start a shell providing GNU Hello in a chroot store:
+
+  ```console
+  # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash
+  ```
+
+  Note that it's necessary to specify `bash` explicitly because your
+  default shell (e.g. `/bin/bash`) generally will not exist in the
+  chroot.
+
+# Description
+
+`nix shell` runs a command in an environment in which the `$PATH`
+variable provides the specified *installables*. If not command is
+specified, it starts the default shell of your user account.
+
+)""

From 9dcd0aebc59a53c622d709c33d4c6f5e20bc0ac7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 17:34:45 +0100
Subject: [PATCH 587/882] generate-manpage.nix: Fix short names

---
 doc/manual/generate-manpage.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index fbd7f3e7d..9f30c8fbc 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -43,7 +43,7 @@ let
         if flag.category or "" != "config"
         then
           "  - `--${longName}`"
-          + (if flag ? shortName then " / `${flag.shortName}`" else "")
+          + (if flag ? shortName then " / `-${flag.shortName}`" else "")
           + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
           + "  \n"
           + "    " + flag.description + "\n\n"

From 28ee307fd8d38e8f6c8dd9d2575435036f3612cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 17:49:58 +0100
Subject: [PATCH 588/882] Add 'nix copy' manpage

---
 src/nix/copy.cc | 29 ++++---------------------
 src/nix/copy.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 25 deletions(-)
 create mode 100644 src/nix/copy.md

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index cb31aac8f..2394eb46d 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -54,32 +54,11 @@ struct CmdCopy : StorePathsCommand
         return "copy paths between Nix stores";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To copy Firefox from the local store to a binary cache in file:///tmp/cache:",
-                "nix copy --to file:///tmp/cache $(type -p firefox)"
-            },
-            Example{
-                "To copy the entire current NixOS system closure to another machine via SSH:",
-                "nix copy --to ssh://server /run/current-system"
-            },
-            Example{
-                "To copy a closure from another machine via SSH:",
-                "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
-            },
-#ifdef ENABLE_S3
-            Example{
-                "To copy Hello to an S3 binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello"
-            },
-            Example{
-                "To copy Hello to an S3-compatible binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello"
-            },
-#endif
-        };
+        return
+          #include "copy.md"
+          ;
     }
 
     Category category() override { return catSecondary; }
diff --git a/src/nix/copy.md b/src/nix/copy.md
new file mode 100644
index 000000000..25e0ddadc
--- /dev/null
+++ b/src/nix/copy.md
@@ -0,0 +1,58 @@
+R""(
+
+# Examples
+
+* Copy Firefox from the local store to a binary cache in `/tmp/cache`:
+
+  ```console
+  # nix copy --to file:///tmp/cache $(type -p firefox)
+  ```
+
+  Note the `file://` - without this, the destination is a chroot
+  store, not a binary cache.
+
+* Copy the entire current NixOS system closure to another machine via
+  SSH:
+
+  ```console
+  # nix copy -s --to ssh://server /run/current-system
+  ```
+
+  The `-s` flag causes the remote machine to try to substitute missing
+  store paths, which may be faster if the link between the local and
+  remote machines is slower than the link between the remote machine
+  and its substituters (e.g. `https://cache.nixos.org`).
+
+* Copy a closure from another machine via SSH:
+
+  ```console
+  # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2
+  ```
+
+* Copy Hello to a binary cache in an Amazon S3 bucket:
+
+  ```console
+  # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello
+  ```
+
+  or to an S3-compatible storage system:
+
+  ```console
+  # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello
+  ```
+
+  Note that this only works if Nix is built with AWS support.
+
+* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`:
+
+  ```console
+  # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs
+  ```
+
+# Description
+
+`nix copy` copies store path closures between two Nix stores. The
+source store is specified using `--from` and the destination using
+`--to`. If one of these is omitted, it defaults to the local store.
+
+)""

From e9de689a6efca961099f1acfc574009f31a6f130 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 18:09:30 +0100
Subject: [PATCH 589/882] Add 'nix search' manpage

---
 src/nix/search.cc | 25 ++++------------
 src/nix/search.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 20 deletions(-)
 create mode 100644 src/nix/search.md

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 47770e128..9f864b3a4 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -41,29 +41,14 @@ struct CmdSearch : InstallableCommand, MixJSON
 
     std::string description() override
     {
-        return "query available packages";
+        return "search for packages";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show all packages in the flake in the current directory:",
-                "nix search"
-            },
-            Example{
-                "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:",
-                "nix search nixpkgs blender"
-            },
-            Example{
-                "To search for Firefox or Chromium:",
-                "nix search nixpkgs 'firefox|chromium'"
-            },
-            Example{
-                "To search for packages containing 'git' and either 'frontend' or 'gui':",
-                "nix search nixpkgs git 'frontend|gui'"
-            }
-        };
+        return
+          #include "search.md"
+          ;
     }
 
     Strings getDefaultFlakeAttrPaths() override
diff --git a/src/nix/search.md b/src/nix/search.md
new file mode 100644
index 000000000..d182788a6
--- /dev/null
+++ b/src/nix/search.md
@@ -0,0 +1,72 @@
+R""(
+
+# Examples
+
+* Show all packages in the `nixpkgs` flake:
+
+  ```console
+  # nix search nixpkgs
+  * legacyPackages.x86_64-linux.AMB-plugins (0.8.1)
+    A set of ambisonics ladspa plugins
+
+  * legacyPackages.x86_64-linux.ArchiSteamFarm (4.3.1.0)
+    Application with primary purpose of idling Steam cards from multiple accounts simultaneously
+  …
+  ```
+
+* Show packages in the `nixpkgs` flake containing `blender` in its
+  name or description:
+
+  ```console
+  # nix search nixpkgs blender
+  * legacyPackages.x86_64-linux.blender (2.91.0)
+    3D Creation/Animation/Publishing System
+  ```
+
+* Search for packages underneath the attribute `gnome3` in Nixpkgs:
+
+  ```console
+  # nix search nixpkgs#gnome3 vala
+  * legacyPackages.x86_64-linux.gnome3.vala (0.48.9)
+    Compiler for GObject type system
+  ```
+
+* Show all packages in the flake in the current directory:
+
+  ```console
+  # nix search
+  ```
+
+* Search for Firefox or Chromium:
+
+  ```console
+  # nix search nixpkgs 'firefox|chromium'
+  ```
+
+* Search for packages containing `git'`and either `frontend` or `gui`:
+
+  ```console
+  # nix search nixpkgs git 'frontend|gui'
+  ```
+
+# Description
+
+`nix search` searches *installable* (which must be evaluatable, e.g. a
+flake) for packages whose name or description matches all of the
+regular expressions *regex*.  For each matching package, It prints the
+full attribute name (from the root of the installable), the version
+and the `meta.description` field, highlighting the substrings that
+were matched by the regular expressions. If no regular expressions are
+specified, all packages are shown.
+
+# Flake output attributes
+
+If no flake output attribute is given, `nix search` searches for
+packages:
+
+* Directly underneath `packages.<system>`.
+
+* Underneath `legacyPackages.<system>`, recursing into attribute sets
+  that contain an attribute `recurseForDerivations = true`.
+
+)""

From 42cc98f8d66627ff7e396fb809034d3389b3bd0a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 21:52:22 +0100
Subject: [PATCH 590/882] Add 'nix develop' and `nix print-dev-env' manpages

---
 src/nix/develop.cc       | 38 ++++------------
 src/nix/develop.md       | 94 ++++++++++++++++++++++++++++++++++++++++
 src/nix/print-dev-env.md | 19 ++++++++
 3 files changed, 121 insertions(+), 30 deletions(-)
 create mode 100644 src/nix/develop.md
 create mode 100644 src/nix/print-dev-env.md

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 457d94382..edd87f246 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -385,30 +385,11 @@ struct CmdDevelop : Common, MixEnvironment
         return "run a bash shell that provides the build environment of a derivation";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To get the build environment of GNU hello:",
-                "nix develop nixpkgs#hello"
-            },
-            Example{
-                "To get the build environment of the default package of flake in the current directory:",
-                "nix develop"
-            },
-            Example{
-                "To store the build environment in a profile:",
-                "nix develop --profile /tmp/my-shell nixpkgs#hello"
-            },
-            Example{
-                "To use a build environment previously recorded in a profile:",
-                "nix develop /tmp/my-shell"
-            },
-            Example{
-                "To replace all occurences of a store path with a writable directory:",
-                "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev"
-            },
-        };
+        return
+          #include "develop.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -495,14 +476,11 @@ struct CmdPrintDevEnv : Common
         return "print shell code that can be sourced by bash to reproduce the build environment of a derivation";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To apply the build environment of GNU hello to the current shell:",
-                ". <(nix print-dev-env nixpkgs#hello)"
-            },
-        };
+        return
+          #include "print-dev-env.md"
+          ;
     }
 
     Category category() override { return catUtility; }
diff --git a/src/nix/develop.md b/src/nix/develop.md
new file mode 100644
index 000000000..7a906d10d
--- /dev/null
+++ b/src/nix/develop.md
@@ -0,0 +1,94 @@
+R""(
+
+# Examples
+
+* Start a shell with the build environment of the default package of
+  the flake in the current directory:
+
+  ```console
+  # nix develop
+  ```
+
+  Typical commands to run inside this shell are:
+
+  ```console
+  # configurePhase
+  # buildPhase
+  # installPhase
+  ```
+
+  Alternatively, you can run whatever build tools your project uses
+  directly, e.g. for a typical Unix project:
+
+  ```console
+  # ./configure --prefix=$out
+  # make
+  # make install
+  ```
+
+* Run a particular build phase directly:
+
+  ```console
+  # nix develop --configure
+  # nix develop --build
+  # nix develop --check
+  # nix develop --install
+  # nix develop --installcheck
+  ```
+
+* Start a shell with the build environment of GNU Hello:
+
+  ```console
+  # nix develop nixpkgs#hello
+  ```
+
+* Record a build environment in a profile:
+
+  ```console
+  # nix develop --profile /tmp/my-build-env nixpkgs#hello
+  ```
+
+* Use a build environment previously recorded in a profile:
+
+  ```cosnole
+  # nix develop /tmp/my-build-env
+  ```
+
+* Replace all occurences of the store path corresponding to
+  `glibc.dev` with a writable directory:
+
+  ```console
+  # nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev
+  ```
+
+  Note that this is useful if you're running a `nix develop` shell for
+  `nixpkgs#glibc` in `~/my-glibc` and want to compile another package
+  against it.
+
+# Description
+
+`nix develop` starts a `bash` shell that provides an interactive build
+environment nearly identical to what Nix would use to build
+*installable*. Inside this shell, environment variables and shell
+functions are set up so that you can interactively and incrementally
+build your package.
+
+Nix determines the build environment by building a modified version of
+the derivation *installable* that just records the environment
+initialised by `stdenv` and exits. This build environment can be
+recorded into a profile using `--profile`.
+
+The prompt used by the `bash` shell can be customised by setting the
+`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in
+the flake's `nixConfig` attribute.
+
+# Flake output attributes
+
+If no flake output attribute is given, `nix run` tries the following
+flake output attributes:
+
+* `devShell.<system>`
+
+* `defaultPackage.<system>`
+
+)""
diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md
new file mode 100644
index 000000000..b80252acf
--- /dev/null
+++ b/src/nix/print-dev-env.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* Apply the build environment of GNU hello to the current shell:
+
+  ```console
+  # . <(nix print-dev-env nixpkgs#hello)
+  ```
+
+# Description
+
+This command prints a shell script that can be sourced by `b`ash and
+that sets the environment variables and shell functions defined by the
+build process of *installable*. This allows you to get a similar build
+environment in your current shell rather than in a subshell (as with
+`nix develop`).
+
+)""

From e90e74523238f37748d9f406732919374d7ee561 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 22:57:14 +0100
Subject: [PATCH 591/882] Add 'nix registry' manpages

This also documents the registry format and matching/unification
semantics (though not quite correctly).
---
 src/nix/registry-add.md    | 33 +++++++++++++
 src/nix/registry-list.md   | 29 +++++++++++
 src/nix/registry-pin.md    | 38 +++++++++++++++
 src/nix/registry-remove.md | 16 +++++++
 src/nix/registry.cc        | 35 ++++++++++++++
 src/nix/registry.md        | 98 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 249 insertions(+)
 create mode 100644 src/nix/registry-add.md
 create mode 100644 src/nix/registry-list.md
 create mode 100644 src/nix/registry-pin.md
 create mode 100644 src/nix/registry-remove.md
 create mode 100644 src/nix/registry.md

diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md
new file mode 100644
index 000000000..80a31996a
--- /dev/null
+++ b/src/nix/registry-add.md
@@ -0,0 +1,33 @@
+R""(
+
+# Examples
+
+* Set the `nixpkgs` flake identifier to a specific branch of Nixpkgs:
+
+  ```console
+  # nix registry add nixpkgs github:NixOS/nixpkgs/nixos-20.03
+  ```
+
+* Pin `nixpkgs` to a specific revision:
+
+  ```console
+  # nix registry add nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+  ```
+
+* Add an entry that redirects a specific branch of `nixpkgs` to
+  another fork:
+
+  ```console
+  # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs
+  ```
+
+# Description
+
+This command adds an entry to the user registry that maps flake
+reference *from-url* to flake reference *to-url*. If an entry for
+*from-url* already exists, it is overwritten.
+
+Entries can be removed using [`nix registry
+remove`](./nix3-registry-remove.md).
+
+)""
diff --git a/src/nix/registry-list.md b/src/nix/registry-list.md
new file mode 100644
index 000000000..30b6e29d8
--- /dev/null
+++ b/src/nix/registry-list.md
@@ -0,0 +1,29 @@
+R""(
+
+# Examples
+
+* Show the contents of all registries:
+
+  ```console
+  # nix registry list
+  user   flake:dwarffs github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5
+  system flake:nixpkgs path:/nix/store/fxl9mrm5xvzam0lxi9ygdmksskx4qq8s-source?lastModified=1605220118&narHash=sha256-Und10ixH1WuW0XHYMxxuHRohKYb45R%2fT8CwZuLd2D2Q=&rev=3090c65041104931adda7625d37fa874b2b5c124
+  global flake:blender-bin github:edolstra/nix-warez?dir=blender
+  global flake:dwarffs github:edolstra/dwarffs
+  …
+  ```
+
+# Description
+
+This command displays the contents of all registries on standard
+output. Each line represents one registry entry in the format *type*
+*from* *to*, where *type* denotes the registry containing the entry:
+
+* `flags`: entries specified on the command line using `--override-flake`.
+* `user`: the user registry.
+* `system`: the system registry.
+* `global`: the global registry.
+
+See the [`nix registry` manual page](./nix3-registry.md) for more details.
+
+)""
diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md
new file mode 100644
index 000000000..6e97e003e
--- /dev/null
+++ b/src/nix/registry-pin.md
@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Pin `nixpkgs` to its most recent Git revision:
+
+  ```console
+  # nix registry pin nixpkgs
+  ```
+
+  Afterwards the user registry will have an entry like this:
+
+  ```console
+  nix registry list | grep '^user '
+  user   flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+  ```
+
+  and `nix flake info` will say:
+
+  ```console
+  # nix flake info nixpkgs
+  Resolved URL:  github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+  Locked URL:    github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+  …
+  ```
+
+# Description
+
+This command adds an entry to the user registry that maps flake
+reference *url* to the corresponding *locked* flake reference, that
+is, a flake reference that specifies an exact revision or content
+hash. This ensures that until this registry entry is removed, all uses
+of *url* will resolve to exactly the same flake.
+
+Entries can be removed using [`nix registry
+remove`](./nix3-registry-remove.md).
+
+)""
diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md
new file mode 100644
index 000000000..4c0eb4947
--- /dev/null
+++ b/src/nix/registry-remove.md
@@ -0,0 +1,16 @@
+R""(
+
+# Examples
+
+* Remove the entry `nixpkgs` from the user registry:
+
+  ```console
+  # nix registry remove nixpkgs
+  ```
+
+# Description
+
+This command removes from the user registry any entry for flake
+reference *url*.
+
+)""
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index 9352e00a7..f9719600f 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -17,6 +17,13 @@ struct CmdRegistryList : StoreCommand
         return "list available Nix flakes";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "registry-list.md"
+          ;
+    }
+
     void run(nix::ref<nix::Store> store) override
     {
         using namespace fetchers;
@@ -47,6 +54,13 @@ struct CmdRegistryAdd : MixEvalArgs, Command
         return "add/replace flake in user flake registry";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "registry-add.md"
+          ;
+    }
+
     CmdRegistryAdd()
     {
         expectArg("from-url", &fromUrl);
@@ -75,6 +89,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
         return "remove flake from user flake registry";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "registry-remove.md"
+          ;
+    }
+
     CmdRegistryRemove()
     {
         expectArg("url", &url);
@@ -97,6 +118,13 @@ struct CmdRegistryPin : virtual Args, EvalCommand
         return "pin a flake to its current version in user flake registry";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "registry-pin.md"
+          ;
+    }
+
     CmdRegistryPin()
     {
         expectArg("url", &url);
@@ -132,6 +160,13 @@ struct CmdRegistry : virtual NixMultiCommand
         return "manage the flake registry";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "registry.md"
+          ;
+    }
+
     Category category() override { return catSecondary; }
 
     void run() override
diff --git a/src/nix/registry.md b/src/nix/registry.md
new file mode 100644
index 000000000..557e5795b
--- /dev/null
+++ b/src/nix/registry.md
@@ -0,0 +1,98 @@
+R""(
+
+# Description
+
+`nix flake` provides subcommands for managing *flake
+registries*. Flake registries are a convenience feature that allows
+you to refer to flakes using symbolic identifiers such as `nixpkgs`,
+rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You
+can use these identifiers on the command line (e.g. when you do `nix
+run nixpkgs#hello`) or in flake input specifications in `flake.nix`
+files. The latter are automatically resolved to full URLs and recorded
+in the flake's `flake.lock` file.
+
+In addition, the flake registry allows you to redirect arbitrary flake
+references (e.g. `github:NixOS/patchelf`) to another location, such as
+a local fork.
+
+There are multiple registries. These are, in order from lowest to
+highest precedence:
+
+* The global registry, which is a file downloaded from the URL
+  specified by the setting `flake-registry`. It is cached locally and
+  updated automatically when it's older than `tarball-ttl`
+  seconds. The default global registry is kept in [a GitHub
+  repository](https://github.com/NixOS/flake-registry).
+
+* The system registry, which is shared by all users. The default
+  location is `/etc/nix/registry.json`. On NixOS, the system registry
+  can be specified using the NixOS option `nix.registry`.
+
+* The user registry `~/.config/nix/registry.json`. This registry can
+  be modified by commands such as `nix flake pin`.
+
+* Overrides specified on the command line using the option
+  `--override-flake`.
+
+# Registry format
+
+A registry is a JSON file with the following format:
+
+```json
+{
+  "version": 2,
+  [
+    {
+      "from": {
+        "type": "indirect",
+        "id": "nixpkgs"
+      },
+      "to": {
+        "type": "github",
+        "owner": "NixOS",
+        "repo": "nixpkgs"
+      }
+    },
+    ...
+  ]
+}
+```
+
+That is, it contains a list of objects with attributes `from` and
+`to`, both of which contain a flake reference in attribute
+representation. (For example, `{"type": "indirect", "id": "nixpkgs"}`
+is the attribute representation of `nixpkgs`, while `{"type":
+"github", "owner": "NixOS", "repo": "nixpkgs"}` is the attribute
+representation of `github:NixOS/nixpkgs`.)
+
+Given some flake reference *R*, a registry entry is used if its
+`from` flake reference *matches* *R*. *R* is then replaced by the
+*unification* of the `to` flake reference with *R*.
+
+# Matching
+
+The `from` flake reference in a registry entry *matches* some flake
+reference *R* if the attributes in `from` are the same as the
+attributes in `R`. For example:
+
+* `nixpkgs` matches with `nixpkgs`.
+
+* `nixpkgs` matches with `nixpkgs/nixos-20.09`.
+
+* `nixpkgs/nixos-20.09` does not match with `nixpkgs`.
+
+* `nixpkgs` does not match with `git://github.com/NixOS/patchelf`.
+
+# Unification
+
+The `to` flake reference in a registry entry is *unified* with some flake
+reference *R* by taking `to` and applying the `rev` and `ref`
+attributes from *R*, if specified. For example:
+
+* `github:NixOS/nixpkgs` unified with `nixpkgs` produces `github:NixOS/nixpkgs`.
+
+* `github:NixOS/nixpkgs` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`.
+
+* `github:NixOS/nixpkgs/master` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`.
+
+)""

From 4e065229c75455d3cee8bc9e791c10faa93c1b50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Dec 2020 23:01:50 +0100
Subject: [PATCH 592/882] Typo

---
 src/nix/develop.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/develop.md b/src/nix/develop.md
index 7a906d10d..e71d9f8aa 100644
--- a/src/nix/develop.md
+++ b/src/nix/develop.md
@@ -50,7 +50,7 @@ R""(
 
 * Use a build environment previously recorded in a profile:
 
-  ```cosnole
+  ```console
   # nix develop /tmp/my-build-env
   ```
 

From b2262be19babc37a54bed4384fecba11d87f7364 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 12:55:24 +0100
Subject: [PATCH 593/882] Add 'nix edit' manpage

---
 src/nix/edit.cc | 11 ++++-------
 src/nix/edit.md | 31 +++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 7 deletions(-)
 create mode 100644 src/nix/edit.md

diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 51c16f5a9..6472dd27a 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -15,14 +15,11 @@ struct CmdEdit : InstallableCommand
         return "open the Nix expression of a Nix package in $EDITOR";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To open the Nix expression of the GNU Hello package:",
-                "nix edit nixpkgs#hello"
-            },
-        };
+        return
+          #include "edit.md"
+          ;
     }
 
     Category category() override { return catSecondary; }
diff --git a/src/nix/edit.md b/src/nix/edit.md
new file mode 100644
index 000000000..80563d06b
--- /dev/null
+++ b/src/nix/edit.md
@@ -0,0 +1,31 @@
+R""(
+
+# Examples
+
+* Open the Nix expression of the GNU Hello package:
+
+  ```console
+  # nix edit nixpkgs#hello
+  ```
+
+* Get the filename and line number used by `nix edit`:
+
+  ```console
+  # nix eval --raw nixpkgs#hello.meta.position
+  /nix/store/fvafw0gvwayzdan642wrv84pzm5bgpmy-source/pkgs/applications/misc/hello/default.nix:15
+  ```
+
+# Description
+
+This command opens the Nix expression of a derivation in an
+editor. The filename and line number of the derivation are taken from
+its `meta.position` attribute. Nixpkgs' `stdenv.mkDerivation` sets
+this attribute to the location of the definition of the
+`meta.description`, `version` or `name` derivation attributes.
+
+The editor to invoke is specified by the `EDITOR` environment
+variable. It defaults to `cat`. If the editor is `emacs`, `nano` or
+`vim`, it is passed the line number of the derivation using the
+argument `+<lineno>`.
+
+)""

From 6ce393392b4e303f51a512d64627cbd68a73c0e7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 13:07:01 +0100
Subject: [PATCH 594/882] Add 'nix repl' manpage

---
 src/nix/repl.cc | 12 +++++------
 src/nix/repl.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 7 deletions(-)
 create mode 100644 src/nix/repl.md

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index a992d8732..bce8d31dc 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -405,6 +405,7 @@ bool NixRepl::processLine(string line)
     }
 
     if (command == ":?" || command == ":help") {
+        // FIXME: convert to Markdown, include in the 'nix repl' manpage.
         std::cout
              << "The following commands are available:\n"
              << "\n"
@@ -801,14 +802,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         return "start an interactive environment for evaluating Nix expressions";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-          Example{
-            "Display all special commands within the REPL:",
-            "nix repl\nnix-repl> :?"
-          }
-        };
+        return
+          #include "repl.md"
+          ;
     }
 
     void run(ref<Store> store) override
diff --git a/src/nix/repl.md b/src/nix/repl.md
new file mode 100644
index 000000000..bba60f871
--- /dev/null
+++ b/src/nix/repl.md
@@ -0,0 +1,57 @@
+R""(
+
+# Examples
+
+* Display all special commands within the REPL:
+
+  ```console
+  # nix repl
+  nix-repl> :?
+  ```
+
+* Evaluate some simple Nix expressions:
+
+  ```console
+  # nix repl
+
+  nix-repl> 1 + 2
+  3
+
+  nix-repl> map (x: x * 2) [1 2 3]
+  [ 2 4 6 ]
+  ```
+
+* Interact with Nixpkgs in the REPL:
+
+  ```console
+  # nix repl '<nixpkgs>'
+
+  Loading '<nixpkgs>'...
+  Added 12428 variables.
+
+  nix-repl> emacs.name
+  "emacs-27.1"
+
+  nix-repl> emacs.drvPath
+  "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv"
+
+  nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out"
+
+  nix-repl> :b x
+  this derivation produced the following outputs:
+    out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello
+
+  nix-repl> builtins.readFile drv
+  "Hello, world!\n"
+  ```
+
+# Description
+
+This command provides an interactive environment for evaluating Nix
+expressions. (REPL stands for 'read–eval–print loop'.)
+
+On startup, it loads the Nix expressions named *files* and adds them
+into the lexical scope. You can load addition files using the `:l
+<filename>` command, or reload all files using `:r`.
+
+)""

From 58bacc85e79b318f28e456a72be2d6a7c8d86991 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 13:41:47 +0100
Subject: [PATCH 595/882] Add 'nix log' manpage

---
 src/nix/log.cc | 19 ++++---------------
 src/nix/log.md | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 15 deletions(-)
 create mode 100644 src/nix/log.md

diff --git a/src/nix/log.cc b/src/nix/log.cc
index 33a3053f5..67d3742d6 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -13,22 +13,11 @@ struct CmdLog : InstallableCommand
         return "show the build log of the specified packages or paths, if available";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To get the build log of GNU Hello:",
-                "nix log nixpkgs#hello"
-            },
-            Example{
-                "To get the build log of a specific path:",
-                "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1"
-            },
-            Example{
-                "To get a build log from a specific binary cache:",
-                "nix log --store https://cache.nixos.org nixpkgs#hello"
-            },
-        };
+        return
+          #include "log.md"
+          ;
     }
 
     Category category() override { return catSecondary; }
diff --git a/src/nix/log.md b/src/nix/log.md
new file mode 100644
index 000000000..8ee2f1d19
--- /dev/null
+++ b/src/nix/log.md
@@ -0,0 +1,40 @@
+R""(
+
+# Examples
+
+* Get the build log of GNU Hello:
+
+  ```console
+  # nix log nixpkgs#hello
+  ```
+
+* Get the build log of a specific store path:
+
+  ```console
+  # nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1
+  ```
+
+* Get a build log from a specific binary cache:
+
+  ```console
+  # nix log --store https://cache.nixos.org nixpkgs#hello
+  ```
+
+# Description
+
+This command prints the log of a previous build of a derivation on
+standard output.
+
+Nix looks for build logs in two places:
+
+* In the directory `/nix/var/log/nix/drvs`, which contains logs for
+  locally built derivations.
+
+* In the binary caches listed in the `substituters` setting. Logs
+  should be named `<cache>/log/<base-name-of-store-path>`, where
+  `store-path` is a derivation,
+  e.g. `https://cache.nixos.org/log/dvmig8jgrdapvbyxb1rprckdmdqx08kv-hello-2.10.drv`.
+  For non-derivation store paths, Nix will first try to determine the
+  deriver by fetching the `.narinfo` file for this store path.
+
+)""

From f34b1801a4ec9060504a6b712b52763e88b4341f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 17:41:58 +0100
Subject: [PATCH 596/882] Tweak

---
 src/nix/log.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/log.md b/src/nix/log.md
index 8ee2f1d19..1c76226a3 100644
--- a/src/nix/log.md
+++ b/src/nix/log.md
@@ -22,8 +22,8 @@ R""(
 
 # Description
 
-This command prints the log of a previous build of a derivation on
-standard output.
+This command prints the log of a previous build of the derivation
+*installable* on standard output.
 
 Nix looks for build logs in two places:
 

From 53ce20eab779e0dd156af2ac9ec2318814edc7cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 17:55:59 +0100
Subject: [PATCH 597/882] Add 'nix store ping' manpage

---
 src/nix/ping-store.cc | 13 +++++--------
 src/nix/ping-store.md | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 8 deletions(-)
 create mode 100644 src/nix/ping-store.md

diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 19b1a55c8..62b645b06 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -8,17 +8,14 @@ struct CmdPingStore : StoreCommand
 {
     std::string description() override
     {
-        return "test whether a store can be opened";
+        return "test whether a store can be accessed";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To test whether connecting to a remote Nix store via SSH works:",
-                "nix store ping --store ssh://mac1"
-            },
-        };
+        return
+          #include "ping-store.md"
+          ;
     }
 
     void run(ref<Store> store) override
diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md
new file mode 100644
index 000000000..322093091
--- /dev/null
+++ b/src/nix/ping-store.md
@@ -0,0 +1,30 @@
+R""(
+
+# Examples
+
+* Test whether connecting to a remote Nix store via SSH works:
+
+  ```console
+  # nix store ping --store ssh://mac1
+  ```
+
+* Test whether a URL is a valid binary cache:
+
+  ```console
+  # nix store ping --store https://cache.nixos.org
+  ```
+
+* Test whether the Nix daemon is up and running:
+
+  ```console
+  # nix store ping --store daemon
+  ```
+
+# Description
+
+This command tests whether a particular Nix store (specified by the
+argument `--store` *url*) can be accessed. What this means is
+dependent on the type of the store. For instance, for an SSH store it
+means that Nix can connect to the specified machine.
+
+)""

From a407d14339c2c480f0103a501bcd8a3373d935cb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 18:34:52 +0100
Subject: [PATCH 598/882] Add 'nix eval' manpage

---
 src/nix/eval.cc | 27 +++---------------
 src/nix/eval.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 23 deletions(-)
 create mode 100644 src/nix/eval.md

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index ea82e5300..321df7495 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -40,30 +40,11 @@ struct CmdEval : MixJSON, InstallableCommand
         return "evaluate a Nix expression";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            {
-                "To evaluate a Nix expression given on the command line:",
-                "nix eval --expr '1 + 2'"
-            },
-            {
-                "To evaluate a Nix expression from a file or URI:",
-                "nix eval -f ./my-nixpkgs hello.name"
-            },
-            {
-                "To get the current version of Nixpkgs:",
-                "nix eval --raw nixpkgs#lib.version"
-            },
-            {
-                "To print the store path of the Hello package:",
-                "nix eval --raw nixpkgs#hello"
-            },
-            {
-                "To get a list of checks in the 'nix' flake:",
-                "nix eval nix#checks.x86_64-linux --apply builtins.attrNames"
-            },
-        };
+        return
+          #include "eval.md"
+          ;
     }
 
     Category category() override { return catSecondary; }
diff --git a/src/nix/eval.md b/src/nix/eval.md
new file mode 100644
index 000000000..61334cde1
--- /dev/null
+++ b/src/nix/eval.md
@@ -0,0 +1,74 @@
+R""(
+
+# Examples
+
+* Evaluate a Nix expression given on the command line:
+
+  ```console
+  # nix eval --expr '1 + 2'
+  ```
+
+* Evaluate a Nix expression to JSON:
+
+  ```console
+  # nix eval --json --expr '{ x = 1; }'
+  {"x":1}
+  ```
+
+* Evaluate a Nix expression from a file:
+
+  ```console
+  # nix eval -f ./my-nixpkgs hello.name
+  ```
+
+* Get the current version of the `nixpkgs` flake:
+
+  ```console
+  # nix eval --raw nixpkgs#lib.version
+  ```
+
+* Print the store path of the Hello package:
+
+  ```console
+  # nix eval --raw nixpkgs#hello
+  ```
+
+* Get a list of checks in the `nix` flake:
+
+  ```console
+  # nix eval nix#checks.x86_64-linux --apply builtins.attrNames
+  ```
+
+* Generate a directory with the specified contents:
+
+  ```console
+  # nix eval --write-to ./out --expr '{ foo = "bar"; subdir.bla = "123"; }'
+  # cat ./out/foo
+  bar
+  # cat ./out/subdir/bla
+  123
+
+# Description
+
+This command evaluates the Nix expression *installable* and prints the
+result on standard output.
+
+# Output format
+
+`nix eval` can produce output in several formats:
+
+* By default, the evaluation result is printed as a Nix expression.
+
+* With `--json`, the evaluation result is printed in JSON format. Note
+  that this fails if the result contains values that are not
+  representable as JSON, such as functions.
+
+* With `--raw`, the evaluation result must be a string, which is
+  printed verbatim, without any quoting.
+
+* With `--write-to` *path*, the evaluation result must be a string or
+  a nested attribute set whose leaf values are strings. These strings
+  are written to files named *path*/*attrpath*. *path* must not
+  already exist.
+
+)""

From 2cc02bbe7675acb4754b928b5c57fa316600e877 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 19:21:48 +0100
Subject: [PATCH 599/882] Add 'nix nar' manpages

---
 src/nix/cat.cc           |  7 +++++++
 src/nix/dump-path.cc     | 11 ++++-------
 src/nix/ls.cc            | 13 ++++++-------
 src/nix/nar-cat.md       | 19 +++++++++++++++++++
 src/nix/nar-dump-path.md | 17 +++++++++++++++++
 src/nix/nar-ls.md        | 24 ++++++++++++++++++++++++
 src/nix/nar.cc           |  9 ++++++++-
 src/nix/nar.md           | 13 +++++++++++++
 8 files changed, 98 insertions(+), 15 deletions(-)
 create mode 100644 src/nix/nar-cat.md
 create mode 100644 src/nix/nar-dump-path.md
 create mode 100644 src/nix/nar-ls.md
 create mode 100644 src/nix/nar.md

diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 2ecffc9a5..fe2f0a241 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -62,6 +62,13 @@ struct CmdCatNar : StoreCommand, MixCat
         return "print the contents of a file inside a NAR file on stdout";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "nar-cat.md"
+          ;
+    }
+
     void run(ref<Store> store) override
     {
         cat(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 256db64a9..63393ef9c 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -49,14 +49,11 @@ struct CmdDumpPath2 : Command
         return "serialise a path to stdout in NAR format";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To serialise directory 'foo' as a NAR:",
-                "nix nar dump-path ./foo"
-            },
-        };
+        return
+          #include "nar-dump-path.md"
+          ;
     }
 
     void run() override
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 1f5ed6913..df67240f9 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -75,6 +75,8 @@ struct MixLs : virtual Args, MixJSON
 
         if (json) {
             JSONPlaceholder jsonRoot(std::cout);
+            if (showDirectory)
+                throw UsageError("'--directory' is useless with '--json'");
             listNar(jsonRoot, accessor, path, recursive);
         } else
             listText(accessor);
@@ -127,14 +129,11 @@ struct CmdLsNar : Command, MixLs
         expectArg("path", &path);
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To list a specific file in a NAR:",
-                "nix nar ls -l hello.nar /bin/hello"
-            },
-        };
+        return
+          #include "nar-ls.md"
+          ;
     }
 
     std::string description() override
diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md
new file mode 100644
index 000000000..55c481a28
--- /dev/null
+++ b/src/nix/nar-cat.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* List a file in a NAR and pipe it through `gunzip`:
+
+  ```console
+  # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip
+  .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.46.4.
+  .TH HELLO "1" "November 2014" "hello 2.10" "User Commands"
+  …
+  ```
+
+# Description
+
+This command prints on standard output the contents of the regular
+file *path* inside the NAR file *nar*.
+
+)""
diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md
new file mode 100644
index 000000000..26191ad25
--- /dev/null
+++ b/src/nix/nar-dump-path.md
@@ -0,0 +1,17 @@
+R""(
+
+# Examples
+
+* To serialise directory `foo` as a NAR:
+
+  ```console
+  # nix nar dump-path ./foo > foo.nar
+  ```
+
+# Description
+
+This command generates a NAR file containing the serialisation of
+*path*, which must contain only regular files, directories and
+symbolic links. The NAR is written to standard output.
+
+)""
diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md
new file mode 100644
index 000000000..d373f9715
--- /dev/null
+++ b/src/nix/nar-ls.md
@@ -0,0 +1,24 @@
+R""(
+
+# Examples
+
+* To list a specific file in a NAR:
+
+  ```console
+  # nix nar ls -l ./hello.nar /bin/hello
+  -r-xr-xr-x                38184 hello
+  ```
+
+* To recursively list the contents of a directory inside a NAR, in JSON
+  format:
+
+  ```console
+  # nix nar ls --json -R ./hello.nar /bin
+  {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}}
+  ```
+
+# Description
+
+This command shows information about a *path* inside NAR file *nar*.
+
+)""
diff --git a/src/nix/nar.cc b/src/nix/nar.cc
index e239ce96a..0775d3c25 100644
--- a/src/nix/nar.cc
+++ b/src/nix/nar.cc
@@ -9,7 +9,14 @@ struct CmdNar : NixMultiCommand
 
     std::string description() override
     {
-        return "query the contents of NAR files";
+        return "create or inspect NAR files";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "nar.md"
+          ;
     }
 
     Category category() override { return catUtility; }
diff --git a/src/nix/nar.md b/src/nix/nar.md
new file mode 100644
index 000000000..a83b5c764
--- /dev/null
+++ b/src/nix/nar.md
@@ -0,0 +1,13 @@
+R""(
+
+# Description
+
+`nix nar` provides several subcommands for creating and inspecting
+*Nix Archives* (NARs).
+
+# File format
+
+For the definition of the NAR file format, see Figure 5.2 in
+https://edolstra.github.io/pubs/phd-thesis.pdf.
+
+)""

From c14ed3f8b2cbddb335227d2ff5188896e76b713f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 20:06:19 +0100
Subject: [PATCH 600/882] Add 'nix store' NAR-related manpages

---
 src/nix/cat.cc             |  7 +++++++
 src/nix/dump-path.cc       | 11 ++++-------
 src/nix/ls.cc              | 17 +++++++----------
 src/nix/store-cat.md       | 19 +++++++++++++++++++
 src/nix/store-dump-path.md | 23 +++++++++++++++++++++++
 src/nix/store-ls.md        | 27 +++++++++++++++++++++++++++
 6 files changed, 87 insertions(+), 17 deletions(-)
 create mode 100644 src/nix/store-cat.md
 create mode 100644 src/nix/store-dump-path.md
 create mode 100644 src/nix/store-ls.md

diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index fe2f0a241..e28ee3c50 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -37,6 +37,13 @@ struct CmdCatStore : StoreCommand, MixCat
         return "print the contents of a file in the Nix store on stdout";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "store-cat.md"
+          ;
+    }
+
     void run(ref<Store> store) override
     {
         cat(store->getFSAccessor());
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 63393ef9c..c4edc894b 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -11,14 +11,11 @@ struct CmdDumpPath : StorePathCommand
         return "serialise a store path to stdout in NAR format";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To get a NAR from the binary cache https://cache.nixos.org/:",
-                "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"
-            },
-        };
+        return
+          #include "store-dump-path.md"
+          ;
     }
 
     void run(ref<Store> store, const StorePath & storePath) override
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index df67240f9..d48287f27 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -94,21 +94,18 @@ struct CmdLsStore : StoreCommand, MixLs
         });
     }
 
-    Examples examples() override
-    {
-        return {
-            Example{
-                "To list the contents of a store path in a binary cache:",
-                "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"
-            },
-        };
-    }
-
     std::string description() override
     {
         return "show information about a path in the Nix store";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "store-ls.md"
+          ;
+    }
+
     void run(ref<Store> store) override
     {
         list(store->getFSAccessor());
diff --git a/src/nix/store-cat.md b/src/nix/store-cat.md
new file mode 100644
index 000000000..da2073473
--- /dev/null
+++ b/src/nix/store-cat.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* Show the contents of a file in a binary cache:
+
+  ```console
+  # nix store cat --store https://cache.nixos.org/ \
+      /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello | hexdump -C | head -n1
+  00000000  7f 45 4c 46 02 01 01 00  00 00 00 00 00 00 00 00  |.ELF............|
+  ```
+
+# Description
+
+This command prints on standard output the contents of the regular
+file *path* in a Nix store. *path* can be a top-level store path or
+any file inside a store path.
+
+)""
diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md
new file mode 100644
index 000000000..4ef563526
--- /dev/null
+++ b/src/nix/store-dump-path.md
@@ -0,0 +1,23 @@
+R""(
+
+# Examples
+
+* To get a NAR containing the GNU Hello package:
+
+  ```console
+  # nix store dump-path nixpkgs#hello > hello.nar
+  ```
+
+* To get a NAR from the binary cache https://cache.nixos.org/:
+
+  ```console
+  # nix store dump-path --store https://cache.nixos.org/ \
+      /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25 > glibc.nar
+  ```
+
+# Description
+
+This command generates a NAR file containing the serialisation of the
+store path *installable*. The NAR is written to standard output.
+
+)""
diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md
new file mode 100644
index 000000000..836efce42
--- /dev/null
+++ b/src/nix/store-ls.md
@@ -0,0 +1,27 @@
+R""(
+
+# Examples
+
+* To list the contents of a store path in a binary cache:
+
+  ```console
+  # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
+  dr-xr-xr-x                    0 ./bin
+  -r-xr-xr-x                38184 ./bin/hello
+  dr-xr-xr-x                    0 ./share
+  …
+  ```
+
+* To show information about a specific file in a binary cache:
+
+  ```console
+  # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello
+  -r-xr-xr-x                38184 hello
+  ```
+
+# Description
+
+This command shows information about *path* in a Nix store. *path* can
+be a top-level store path or any file inside a store path.
+
+)""

From 19540744ad164c13803048d5e8883fe68f9d2d10 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 20:33:53 +0100
Subject: [PATCH 601/882] Add 'nix why-depends' manpage

---
 src/nix/why-depends.cc | 19 +++-------
 src/nix/why-depends.md | 80 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 15 deletions(-)
 create mode 100644 src/nix/why-depends.md

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 57b9a2208..297b638cc 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -50,22 +50,11 @@ struct CmdWhyDepends : SourceExprCommand
         return "show why a package has another package in its closure";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show one path through the dependency graph leading from Hello to Glibc:",
-                "nix why-depends nixpkgs#hello nixpkgs#glibc"
-            },
-            Example{
-                "To show all files and paths in the dependency graph leading from Thunderbird to libX11:",
-                "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11"
-            },
-            Example{
-                "To show why Glibc depends on itself:",
-                "nix why-depends nixpkgs#glibc nixpkgs#glibc"
-            },
-        };
+        return
+          #include "why-depends.md"
+          ;
     }
 
     Category category() override { return catSecondary; }
diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md
new file mode 100644
index 000000000..dc13619e1
--- /dev/null
+++ b/src/nix/why-depends.md
@@ -0,0 +1,80 @@
+R""(
+
+# Examples
+
+* Show one path through the dependency graph leading from Hello to
+  Glibc:
+
+  ```console
+  # nix why-depends nixpkgs#hello nixpkgs#glibc
+  /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+  └───bin/hello: …...................../nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32/lib/ld-linux-x86-64.…
+      → /nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32
+  ```
+
+* Show all files and paths in the dependency graph leading from
+  Thunderbird to libX11:
+
+  ```console
+  # nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11
+  /nix/store/qfc8729nzpdln1h0hvi1ziclsl3m84sr-thunderbird-78.5.1
+  ├───lib/thunderbird/libxul.so: …6wrw-libxcb-1.14/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/ssf…
+  │   → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0
+  ├───lib/thunderbird/libxul.so: …pxyc-libXt-1.2.0/lib:/nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5/lib:/nix/store…
+  │   → /nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5
+  │   ├───lib/libXdamage.so.1.1.0: …-libXfixes-5.0.3/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/9l0…
+  │   │   → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0
+  …
+  ```
+
+* Show why Glibc depends on itself:
+
+  ```console
+  # nix why-depends nixpkgs#glibc nixpkgs#glibc
+  /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31
+  └───lib/ld-2.31.so: …che       Do not use /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/etc/ld.so.cache.  --…
+      → /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31
+  ```
+
+* Show why Geeqie has a build-time dependency on `systemd`:
+
+  ```console
+  # nix why-depends --derivation nixpkgs#geeqie nixpkgs#systemd
+  /nix/store/drrpq2fqlrbj98bmazrnww7hm1in3wgj-geeqie-1.4.drv
+  └───/: …atch.drv",["out"]),("/nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv",["dev"]),("/…
+      → /nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv
+      └───/: …16.0.drv",["dev"]),("/nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv",["dev"]),("/nix…
+          → /nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv
+          └───/: ….3.1.drv",["out"]),("/nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv",["dev"]),("/…
+              → /nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv
+  ```
+
+# Description
+
+Nix automatically determines potential runtime dependencies between
+store paths by scanning for the *hash parts* of store paths. For
+instance, if there exists a store path
+`/nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31`, and a file
+inside another store path contains the string `9df65igw…`, then the
+latter store path *refers* to the former, and thus might need it at
+runtime. Nix always maintains the existence of the transitive closure
+of a store path under the references relationship; it is therefore not
+possible to install a store path without having all of its references
+present.
+
+Sometimes Nix packages end up with unexpected runtime dependencies;
+for instance, a reference to a compiler might accidentally end up in a
+binary, causing the former to be in the latter's closure. This kind of
+*closure size bloat* is undesirable.
+
+`nix why-depends` allows you to diagnose the cause of such issues. It
+shows why the store path *package* depends on the store path
+*dependency*, by showing a shortest sequence in the references graph
+from the former to the latter. Also, for each node along this path, it
+shows a file fragment containing a reference to the next store path in
+the sequence.
+
+To show why derivation *package* has a build-time rather than runtime
+dependency on derivation *dependency*, use `--derivation`.
+
+)""

From 6b32551aba5dfd6a912277297eb28cedc92da26d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 21:11:48 +0100
Subject: [PATCH 602/882] Add 'nix upgrade-nix' manpage

---
 src/nix/upgrade-nix.cc | 15 ++++-----------
 src/nix/upgrade-nix.md | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 11 deletions(-)
 create mode 100644 src/nix/upgrade-nix.md

diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 66ecc5b34..79be31e73 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -37,18 +37,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         return "upgrade Nix to the latest stable version";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To upgrade Nix to the latest stable version:",
-                "nix upgrade-nix"
-            },
-            Example{
-                "To upgrade Nix in a specific profile:",
-                "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile"
-            },
-        };
+        return
+          #include "upgrade-nix.md"
+          ;
     }
 
     Category category() override { return catNixInstallation; }
diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md
new file mode 100644
index 000000000..4d27daad9
--- /dev/null
+++ b/src/nix/upgrade-nix.md
@@ -0,0 +1,28 @@
+R""(
+
+# Examples
+
+* Upgrade Nix to the latest stable version:
+
+  ```console
+  # nix upgrade-nix
+  ```
+
+* Upgrade Nix in a specific profile:
+
+  ```console
+  # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile
+  ```
+
+# Description
+
+This command upgrades Nix to the latest version. By default, it
+locates the directory containing the `nix` binary in the `$PATH`
+environment variable. If that directory is a Nix profile, it will
+upgrade the `nix` package in that profile to the latest stable binary
+release.
+
+You cannot use this command to upgrade Nix in the system profile of a
+NixOS system (that is, if `nix` is found in `/run/current-system`).
+
+)""

From 8dd7d7e9db8165c316b1ef168f57ed3632507fe2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 23:45:06 +0100
Subject: [PATCH 603/882] Add 'nix store verify' manpage

---
 src/nix/verify.cc | 15 ++++-----------
 src/nix/verify.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 11 deletions(-)
 create mode 100644 src/nix/verify.md

diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index bcf85d7dd..16d42349f 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -35,18 +35,11 @@ struct CmdVerify : StorePathsCommand
         return "verify the integrity of store paths";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To verify the entire Nix store:",
-                "nix store verify --all"
-            },
-            Example{
-                "To check whether each path in the closure of Firefox has at least 2 signatures:",
-                "nix store verify -r -n2 --no-contents $(type -p firefox)"
-            },
-        };
+        return
+          #include "verify.md"
+          ;
     }
 
     void run(ref<Store> store, StorePaths storePaths) override
diff --git a/src/nix/verify.md b/src/nix/verify.md
new file mode 100644
index 000000000..1c43792e7
--- /dev/null
+++ b/src/nix/verify.md
@@ -0,0 +1,49 @@
+R""(
+
+# Examples
+
+* Verify the entire Nix store:
+
+  ```console
+  # nix store verify --all
+  ```
+
+* Check whether each path in the closure of Firefox has at least 2
+  signatures:
+
+  ```console
+  # nix store verify -r -n2 --no-contents $(type -p firefox)
+  ```
+
+* Verify a store path in the binary cache `https://cache.nixos.org/`:
+
+  ```console
+  # nix store verify --store https://cache.nixos.org/ \
+      /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+  ```
+
+# Description
+
+This command verifies the integrity of the store paths *installables*,
+or, if `--all` is given, the entire Nix store. For each path, it
+checks that
+
+* its contents match the NAR hash recorded in the Nix database; and
+
+* it is *trusted*, that is, it is signed by at least one trusted
+  signing key, is content-addressed, or is built locally ("ultimately
+  trusted").
+
+# Exit status
+
+The exit status of this command is the sum of the following values:
+
+* **1** if any path is corrupted (i.e. its contents don't match the
+  recorded NAR hash).
+
+* **2** if any path is untrusted.
+
+* **4** if any path couldn't be verified for any other reason (such as
+  an I/O error).
+
+)""

From cb25a89f1cb9b3a26d84b3429b309be2cfa513a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Dec 2020 23:54:11 +0100
Subject: [PATCH 604/882] Add 'nix store optimise' manpage

---
 src/nix/optimise-store.cc | 11 ++++-------
 src/nix/optimise-store.md | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+), 7 deletions(-)
 create mode 100644 src/nix/optimise-store.md

diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index bc7f175ac..985006e5a 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -13,14 +13,11 @@ struct CmdOptimiseStore : StoreCommand
         return "replace identical files in the store by hard links";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To optimise the Nix store:",
-                "nix store optimise"
-            },
-        };
+        return
+          #include "optimise-store.md"
+          ;
     }
 
     void run(ref<Store> store) override
diff --git a/src/nix/optimise-store.md b/src/nix/optimise-store.md
new file mode 100644
index 000000000..f6fb66f97
--- /dev/null
+++ b/src/nix/optimise-store.md
@@ -0,0 +1,23 @@
+R""(
+
+# Examples
+
+* Optimise the Nix store:
+
+  ```console
+  nix store optimise
+  ```
+
+# Description
+
+This command deduplicates the Nix store: it scans the store for
+regular files with identical contents, and replaces them with hard
+links to a single instance.
+
+Note that you can also set `auto-optimise-store` to `true` in
+`nix.conf` to perform this optimisation incrementally whenever a new
+path is added to the Nix store. To make this efficient, Nix maintains
+a content-addressed index of all the files in the Nix store in the
+directory `/nix/store/.links/`.
+
+)""

From 2e599dbb88855311c33c70460b82ec16487c9071 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 13:45:42 +0100
Subject: [PATCH 605/882] Add 'nix path-info' manpage

---
 src/nix/path-info.cc | 35 +++--------------
 src/nix/path-info.md | 94 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+), 29 deletions(-)
 create mode 100644 src/nix/path-info.md

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 63cf885f9..30b6a50f8 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -29,38 +29,15 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         return "query information about store paths";
     }
 
-    Category category() override { return catSecondary; }
-
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show the closure sizes of every path in the current NixOS system closure, sorted by size:",
-                "nix path-info -rS /run/current-system | sort -nk2"
-            },
-            Example{
-                "To show a package's closure size and all its dependencies with human readable sizes:",
-                "nix path-info -rsSh nixpkgs#rust"
-            },
-            Example{
-                "To check the existence of a path in a binary cache:",
-                "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/"
-            },
-            Example{
-                "To print the 10 most recently added paths (using --json and the jq(1) command):",
-                "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'"
-            },
-            Example{
-                "To show the size of the entire Nix store:",
-                "nix path-info --json --all | jq 'map(.narSize) | add'"
-            },
-            Example{
-                "To show every path whose closure is bigger than 1 GB, sorted by closure size:",
-                "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'"
-            },
-        };
+        return
+          #include "path-info.md"
+          ;
     }
 
+    Category category() override { return catSecondary; }
+
     void printSize(uint64_t value)
     {
         if (!humanReadable) {
diff --git a/src/nix/path-info.md b/src/nix/path-info.md
new file mode 100644
index 000000000..b4ba5862d
--- /dev/null
+++ b/src/nix/path-info.md
@@ -0,0 +1,94 @@
+R""(
+
+# Examples
+
+* Print the store path produced by `nixpkgs#hello`:
+
+  ```console
+  # nix path-info nixpkgs#hello
+  /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+  ```
+
+* Show the closure sizes of every path in the current NixOS system
+  closure, sorted by size:
+
+  ```console
+  # nix path-info -rS /run/current-system | sort -nk2
+  /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty                                                96
+  /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers                                         112
+  …
+  /nix/store/539jkw9a8dyry7clcv60gk6na816j7y8-etc                                          5783255504
+  /nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65  5887562256
+  ```
+
+* Show a package's closure size and all its dependencies with human
+  readable sizes:
+
+  ```console
+  # nix path-info -rsSh nixpkgs#rustc
+  /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev      386.7K   69.1M
+  /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0          5.9M   37.4M
+  …
+  ```
+
+* Check the existence of a path in a binary cache:
+
+  ```console
+  # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/
+  path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid
+
+  ```
+
+* Print the 10 most recently added paths (using --json and the jq(1)
+  command):
+
+  ```console
+  # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'
+  ```
+
+* Show the size of the entire Nix store:
+
+  ```console
+  # nix path-info --json --all | jq 'map(.narSize) | add'
+  49812020936
+  ```
+
+* Show every path whose closure is bigger than 1 GB, sorted by closure
+  size:
+
+  ```console
+  # nix path-info --json --all -S \
+    | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'
+  [
+    …,
+    [
+      "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65",
+      5887562256
+    ]
+  ]
+  ```
+
+* Print the path of the store derivation produced by `nixpkgs#hello`:
+
+  ```console
+  # nix path-info --derivation nixpkgs#hello
+  /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv
+ ```
+
+# Description
+
+This command shows information about the store paths produced by
+*installables*, or about all paths in the store if you pass `--all`.
+
+By default, this command only prints the store paths. You can get
+additional information by passing flags such as `--closure-size`,
+--size`, `--sigs` or `--json`.
+
+> **Warning**
+>
+> Note that `nix path-info` does not build or substitute the
+> *installables* you specify. Thus, if the corresponding store paths
+> don't already exist, this command will fail. You can use `nix build`
+> to ensure that they exist.
+
+)""

From cdf20e04b7acc0efd7fa9640283103502ac80c53 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 14:14:30 +0100
Subject: [PATCH 606/882] Doh

---
 src/nix/path-info.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/path-info.md b/src/nix/path-info.md
index b4ba5862d..76a83e39d 100644
--- a/src/nix/path-info.md
+++ b/src/nix/path-info.md
@@ -73,7 +73,7 @@ R""(
   ```console
   # nix path-info --derivation nixpkgs#hello
   /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv
- ```
+  ```
 
 # Description
 

From e6bea9c9b10ded0e65981edf84cedd00ec86883a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 17:36:59 +0100
Subject: [PATCH 607/882] Add 'nix store make-content-addressable' manpage

---
 src/nix/make-content-addressable.cc | 17 +++------
 src/nix/make-content-addressable.md | 59 +++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 12 deletions(-)
 create mode 100644 src/nix/make-content-addressable.md

diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 5165c4804..f5bdc7e65 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -15,21 +15,14 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
     std::string description() override
     {
-        return "rewrite a path or closure to content-addressable form";
+        return "rewrite a path or closure to content-addressed form";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To create a content-addressable representation of GNU Hello (but not its dependencies):",
-                "nix store make-content-addressable nixpkgs#hello"
-            },
-            Example{
-                "To compute a content-addressable representation of the current NixOS system closure:",
-                "nix store make-content-addressable -r /run/current-system"
-            },
-        };
+        return
+          #include "make-content-addressable.md"
+          ;
     }
 
     void run(ref<Store> store, StorePaths storePaths) override
diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressable.md
new file mode 100644
index 000000000..3dd847edc
--- /dev/null
+++ b/src/nix/make-content-addressable.md
@@ -0,0 +1,59 @@
+R""(
+
+# Examples
+
+* Create a content-addressed representation of the closure of GNU Hello:
+
+  ```console
+  # nix store make-content-addressable -r nixpkgs#hello
+  …
+  rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
+  ```
+
+  Since the resulting paths are content-addressed, they are always
+  trusted and don't need signatures to copied to another store:
+
+  ```console
+  # nix copy --to /tmp/nix --trusted-public-keys '' /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10
+  ```
+
+  By contrast, the original closure is input-addressed, so it does
+  need signatures to be trusted:
+
+  ```console
+  # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello
+  cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature
+  ```
+
+* Create a content-addressed representation of the current NixOS
+  system closure:
+
+  ```console
+  # nix store make-content-addressable -r /run/current-system
+  ```
+
+# Description
+
+This command converts the closure of the store paths specified by
+*installables* to content-addressed form. Nix store paths are usually
+*input-addressed*, meaning that the hash part of the store path is
+computed from the contents of the derivation (i.e., the build-time
+dependency graph). Input-addressed paths need to be signed by a
+trusted key if you want to import them into a store, because we need
+to trust that the contents of the path were actually built by the
+derivation.
+
+By contrast, in a *content-addressed* path, the hash part is computed
+from the contents of the path. This allows the contents of the path to
+be verified without any additional information such as
+signatures. This means that a command like
+
+```console
+# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \
+    --substituters https://my-cache.example.org
+```
+
+will succeed even if the binary cache `https://my-cache.example.org`
+doesn't present any signatures.
+
+)""

From daf365b0b731bb3ac86128a965394dcff8d6f5b5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 18:40:16 +0100
Subject: [PATCH 608/882] Add 'nix help' manpage

---
 src/nix/help.md | 17 +++++++++++++++++
 src/nix/main.cc | 17 +++++------------
 2 files changed, 22 insertions(+), 12 deletions(-)
 create mode 100644 src/nix/help.md

diff --git a/src/nix/help.md b/src/nix/help.md
new file mode 100644
index 000000000..734f35028
--- /dev/null
+++ b/src/nix/help.md
@@ -0,0 +1,17 @@
+R""(
+
+# Examples
+
+* Show help about `nix` in general:
+
+  ```console
+  # nix help
+  ```
+
+* Show help about a particular subcommand:
+
+  ```console
+      # nix help flake info
+  ```
+
+)""
diff --git a/src/nix/main.cc b/src/nix/main.cc
index e7a15dec9..afe7cb8d7 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -205,21 +205,14 @@ struct CmdHelp : Command
 
     std::string description() override
     {
-        return "show help about 'nix' or a particular subcommand";
+        return "show help about `nix` or a particular subcommand";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show help about 'nix' in general:",
-                "nix help"
-            },
-            Example{
-                "To show help about a particular subcommand:",
-                "nix help run"
-            },
-        };
+        return
+          #include "help.md"
+          ;
     }
 
     void run() override

From 3b123a6ee63aef29d61a3dac7f2f01c2e92cd6d0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 19:13:23 +0100
Subject: [PATCH 609/882] nix show-derivation: Say "system" instead of
 "platform"

There is really no good reason to use "platform" except that that's
what we use internally (also for no good reason).
---
 src/nix/show-derivation.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 8e1a58ac2..211e6a27a 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -103,7 +103,7 @@ struct CmdShowDerivation : InstallablesCommand
                 }
             }
 
-            drvObj.attr("platform", drv.platform);
+            drvObj.attr("system", drv.platform);
             drvObj.attr("builder", drv.builder);
 
             {

From 4f3e7f4eec9ef5fb86aea9f745a3574cc5cfae28 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 19:14:18 +0100
Subject: [PATCH 610/882] Add 'nix show-derivation' manpage

---
 src/nix/show-derivation.cc |  15 ++----
 src/nix/show-derivation.md | 103 +++++++++++++++++++++++++++++++++++++
 2 files changed, 107 insertions(+), 11 deletions(-)
 create mode 100644 src/nix/show-derivation.md

diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 211e6a27a..13f2c8e69 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -29,18 +29,11 @@ struct CmdShowDerivation : InstallablesCommand
         return "show the contents of a store derivation";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show the store derivation that results from evaluating the Hello package:",
-                "nix show-derivation nixpkgs#hello"
-            },
-            Example{
-                "To show the full derivation graph (if available) that produced your NixOS system:",
-                "nix show-derivation -r /run/current-system"
-            },
-        };
+        return
+          #include "show-derivation.md"
+          ;
     }
 
     Category category() override { return catUtility; }
diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md
new file mode 100644
index 000000000..aa863899c
--- /dev/null
+++ b/src/nix/show-derivation.md
@@ -0,0 +1,103 @@
+R""(
+
+# Examples
+
+* Show the store derivation that results from evaluating the Hello
+  package:
+
+  ```console
+  # nix show-derivation nixpkgs#hello
+  {
+    "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": {
+      …
+    }
+  }
+  ```
+
+* Show the full derivation graph (if available) that produced your
+  NixOS system:
+
+  ```console
+  # nix show-derivation -r /run/current-system
+  ```
+
+* Print all files fetched using `fetchurl` by Firefox's dependency
+  graph:
+
+  ```console
+  # nix show-derivation -r nixpkgs#firefox \
+    | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \
+    | uniq | sort
+  ```
+
+  Note that `.outputs.out.hash` selects *fixed-output derivations*
+  (derivations that produce output with a specified content hash),
+  while `.env.urls` selects derivations with a `urls` attribute.
+
+# Description
+
+This command prints on standard output a JSON representation of the
+store derivations to which *installables* evaluate. Store derivations
+are used internally by Nix. They are store paths with extension `.drv`
+that represent the build-time dependency graph to which a Nix
+expression evaluates.
+
+By default, this command only shows top-level derivations, but with
+`--recursive`, it also shows their dependencies.
+
+The JSON output is a JSON object whose keys are the store paths of the
+derivations, and whose values are a JSON object with the following
+fields:
+
+* `outputs`: Information about the output paths of the
+  derivation. This is a JSON object with one member per output, where
+  the key is the output name and the value is a JSON object with these
+  fields:
+
+  * `path`: The output path.
+  * `hashAlgo`: For fixed-output derivations, the hashing algorithm
+    (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a
+    NAR hash rather than a flat file hash.
+  * `hash`: For fixed-output derivations, the expected content hash in
+    base-16.
+
+  Example:
+
+  ```json
+  "outputs": {
+    "out": {
+      "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
+      "hashAlgo": "r:sha256",
+      "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
+    }
+  }
+  ```
+
+* `inputSrcs`: A list of store paths on which this derivation depends.
+
+* `inputDrvs`: A JSON object specifying the derivations on which this
+  derivation depends, and what outputs of those derivations. For
+  example,
+
+  ```json
+  "inputDrvs": {
+    "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+    "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+  }
+  ```
+
+  specifies that this derivation depends on the `dev` output of
+  `curl`, and the `out` output of `unzip`.
+
+* `system`: The system type on which this derivation is to be built
+  (e.g. `x86_64-linux`).
+
+* `builder`: The absolute path of the program to be executed to run
+  the build. Typically this is the `bash` shell
+  (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
+
+* `args`: The command-line arguments passed to the `builder`.
+
+* `env`: The environment passed to the `builder`.
+
+)""

From f4e9d4fcb3e393af2736f28fc41e4e3b79a8e60d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Dec 2020 19:58:04 +0100
Subject: [PATCH 611/882] Add 'nix store diff-closures' manpage

---
 src/nix/diff-closures.cc | 11 ++++-----
 src/nix/diff-closures.md | 51 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 7 deletions(-)
 create mode 100644 src/nix/diff-closures.md

diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc
index f72b5eff7..0c7d531c1 100644
--- a/src/nix/diff-closures.cc
+++ b/src/nix/diff-closures.cc
@@ -121,14 +121,11 @@ struct CmdDiffClosures : SourceExprCommand
         return "show what packages and versions were added and removed between two closures";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            {
-                "To show what got added and removed between two versions of the NixOS system profile:",
-                "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
-            },
-        };
+        return
+          #include "diff-closures.md"
+          ;
     }
 
     void run(ref<Store> store) override
diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md
new file mode 100644
index 000000000..0294c0d8d
--- /dev/null
+++ b/src/nix/diff-closures.md
@@ -0,0 +1,51 @@
+R""(
+
+# Examples
+
+* Show what got added and removed between two versions of the NixOS
+  system profile:
+
+  ```console
+  # nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link
+  acpi-call: 2020-04-07-5.8.16 → 2020-04-07-5.8.18
+  baloo-widgets: 20.08.1 → 20.08.2
+  bluez-qt: +12.6 KiB
+  dolphin: 20.08.1 → 20.08.2, +13.9 KiB
+  kdeconnect: 20.08.2 → ∅, -6597.8 KiB
+  kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB
+  …
+  ```
+
+# Description
+
+This command shows the differences between the two closures *before*
+and *after* with respect to the addition, removal, or version change
+of packages, as well as changes in store path sizes.
+
+For each package name in the two closures (where a package name is
+defined as the name component of a store path excluding the version),
+if there is a change in the set of versions of the package, or a
+change in the size of the store paths of more than 8 KiB, it prints a
+line like this:
+
+```console
+dolphin: 20.08.1 → 20.08.2, +13.9 KiB
+```
+
+No size change is shown if it's below the threshold. If the package
+does not exist in either the *before* or *after* closures, it is
+represented using `∅` (empty set) on the appropriate side of the
+arrow. If a package has an empty version string, the version is
+rendered as `ε` (epsilon).
+
+There may be multiple versions of a package in each closure. In that
+case, only the changed versions are shown. Thus,
+
+```console
+libfoo: 1.2, 1.3 → 1.4
+```
+
+leaves open the possibility that there are other versions (e.g. `1.1`)
+that exist in both closures.
+
+)""

From 0c09f63de84e15c15e8621a53d6d25f023b4ad06 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Dec 2020 11:45:59 +0100
Subject: [PATCH 612/882] Add 'nix bundle' manpage

Fixes #4375.
---
 src/nix/bundle.cc | 11 ++++-------
 src/nix/bundle.md | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 7 deletions(-)
 create mode 100644 src/nix/bundle.md

diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index eddd82f40..5f558b01e 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -40,14 +40,11 @@ struct CmdBundle : InstallableCommand
         return "bundle an application so that it works outside of the Nix store";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To bundle Hello:",
-                "nix bundle hello"
-            },
-        };
+        return
+          #include "bundle.md"
+          ;
     }
 
     Category category() override { return catSecondary; }
diff --git a/src/nix/bundle.md b/src/nix/bundle.md
new file mode 100644
index 000000000..c183a170d
--- /dev/null
+++ b/src/nix/bundle.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Bundle Hello:
+
+  ```console
+  # nix bundle nixpkgs#hello
+  # ./hello
+  Hello, world!
+  ```
+
+* Bundle a specific version of Nix:
+
+  ```console
+  # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec
+  # ./nix --version
+  nix (Nix) 2.4pre20201215_e3ddffb
+  ```
+
+# Description
+
+`nix bundle` packs the closure of the [Nix app](./nix3-run.md)
+*installable* into a single self-extracting executable. See the
+[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle)
+for more details.
+
+> **Note**
+>
+> This command only works on Linux.
+
+)""

From 16e34085e8f45758b97c41cfcd720552c68a3c98 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Dec 2020 14:25:36 +0100
Subject: [PATCH 613/882] Add 'nix profile' manpage

---
 src/nix/profile-diff-closures.md |  28 ++++++++
 src/nix/profile-info.md          |  31 +++++++++
 src/nix/profile-install.md       |  27 ++++++++
 src/nix/profile-remove.md        |  32 +++++++++
 src/nix/profile-upgrade.md       |  41 ++++++++++++
 src/nix/profile.cc               |  90 +++++++++-----------------
 src/nix/profile.md               | 107 +++++++++++++++++++++++++++++++
 7 files changed, 295 insertions(+), 61 deletions(-)
 create mode 100644 src/nix/profile-diff-closures.md
 create mode 100644 src/nix/profile-info.md
 create mode 100644 src/nix/profile-install.md
 create mode 100644 src/nix/profile-remove.md
 create mode 100644 src/nix/profile-upgrade.md
 create mode 100644 src/nix/profile.md

diff --git a/src/nix/profile-diff-closures.md b/src/nix/profile-diff-closures.md
new file mode 100644
index 000000000..295d1252b
--- /dev/null
+++ b/src/nix/profile-diff-closures.md
@@ -0,0 +1,28 @@
+R""(
+
+# Examples
+
+* Show what changed between each version of the NixOS system
+  profile:
+
+  ```console
+  # nix profile diff-closures --profile /nix/var/nix/profiles/system
+  Version 13 -> 14:
+    acpi-call: 2020-04-07-5.8.13 → 2020-04-07-5.8.14
+    aws-sdk-cpp: -6723.1 KiB
+    …
+
+  Version 14 -> 15:
+    acpi-call: 2020-04-07-5.8.14 → 2020-04-07-5.8.16
+    attica: -996.2 KiB
+    breeze-icons: -78713.5 KiB
+    brotli: 1.0.7 → 1.0.9, +44.2 KiB
+  ```
+
+# Description
+
+This command shows the difference between the closures of subsequent
+versions of a profile. See [`nix store
+diff-closures`](nix3-store-diff-closures.md) for details.
+
+)""
diff --git a/src/nix/profile-info.md b/src/nix/profile-info.md
new file mode 100644
index 000000000..a0c04fc8c
--- /dev/null
+++ b/src/nix/profile-info.md
@@ -0,0 +1,31 @@
+R""(
+
+# Examples
+
+* Show what packages are installed in the default profile:
+
+  ```console
+  # nix profile info
+  0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1
+  1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027
+  2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0
+  ```
+
+# Description
+
+This command shows what packages are currently installed in a
+profile. The output consists of one line per package, with the
+following fields:
+
+* An integer that can be used to unambiguously identify the package in
+  invocations of `nix profile remove` and `nix profile upgrade`.
+
+* The original ("mutable") flake reference and output attribute path
+  used at installation time.
+
+* The immutable flake reference to which the mutable flake reference
+  was resolved.
+
+* The store path(s) of the package.
+
+)""
diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md
new file mode 100644
index 000000000..e3009491e
--- /dev/null
+++ b/src/nix/profile-install.md
@@ -0,0 +1,27 @@
+R""(
+
+# Examples
+
+* Install a package from Nixpkgs:
+
+  ```console
+  # nix profile install nixpkgs#hello
+  ```
+
+* Install a package from a specific branch of Nixpkgs:
+
+  ```console
+  # nix profile install nixpkgs/release-20.09#hello
+  ```
+
+* Install a package from a specific revision of Nixpkgs:
+
+  ```console
+  # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello
+  ```
+
+# Description
+
+This command adds *installables* to a Nix profile.
+
+)""
diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md
new file mode 100644
index 000000000..dcf825da9
--- /dev/null
+++ b/src/nix/profile-remove.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Remove a package by position:
+
+  ```console
+  # nix profile remove 3
+  ```
+
+* Remove a package by attribute path:
+
+  ```console
+  # nix profile remove packages.x86_64-linux.hello
+  ```
+
+* Remove all packages:
+  ```console
+  # nix profile remove '.*'
+  ```
+
+* Remove a package by store path:
+
+  ```console
+  # nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10
+  ```
+
+# Description
+
+This command removes a package from a profile.
+
+)""
diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md
new file mode 100644
index 000000000..2bd5d256d
--- /dev/null
+++ b/src/nix/profile-upgrade.md
@@ -0,0 +1,41 @@
+R""(
+
+# Examples
+
+* Upgrade all packages that were installed using a mutable flake
+  reference:
+
+  ```console
+  # nix profile upgrade '.*'
+  ```
+
+* Upgrade a specific package:
+
+  ```console
+  # nix profile upgrade packages.x86_64-linux.hello
+  ```
+
+* Upgrade a specific profile element by number:
+
+  ```console
+  # nix profile info
+  0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify …
+
+  # nix profile upgrade 0
+  ```
+
+# Description
+
+This command upgrades a previously installed package in a Nix profile,
+by fetching and evaluating the latest version of the flake from which
+the package was installed.
+
+> **Warning**
+>
+> This only works if you used a *mutable* flake reference at
+> installation time, e.g. `nixpkgs#hello`. It does not work if you
+> used an *immutable* flake reference
+> (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`),
+> since in that case the "latest version" is always the same.
+
+)""
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 8cf5ccd62..d8d2b3a70 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -151,22 +151,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
         return "install a package into a profile";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To install a package from Nixpkgs:",
-                "nix profile install nixpkgs#hello"
-            },
-            Example{
-                "To install a package from a specific branch of Nixpkgs:",
-                "nix profile install nixpkgs/release-19.09#hello"
-            },
-            Example{
-                "To install a package from a specific revision of Nixpkgs:",
-                "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello"
-            },
-        };
+        return
+          #include "profile-install.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -257,26 +246,11 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
         return "remove packages from a profile";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To remove a package by attribute path:",
-                "nix profile remove packages.x86_64-linux.hello"
-            },
-            Example{
-                "To remove all packages:",
-                "nix profile remove '.*'"
-            },
-            Example{
-                "To remove a package by store path:",
-                "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10"
-            },
-            Example{
-                "To remove a package by position:",
-                "nix profile remove 3"
-            },
-        };
+        return
+          #include "profile-remove.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -310,18 +284,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
         return "upgrade packages using their most recent flake";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To upgrade all packages that were installed using a mutable flake reference:",
-                "nix profile upgrade '.*'"
-            },
-            Example{
-                "To upgrade a specific package:",
-                "nix profile upgrade packages.x86_64-linux.hello"
-            },
-        };
+        return
+          #include "profile-upgrade.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -377,14 +344,11 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
         return "list installed packages";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show what packages are installed in the default profile:",
-                "nix profile info"
-            },
-        };
+        return
+          #include "profile-info.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -405,17 +369,14 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
 {
     std::string description() override
     {
-        return "show the closure difference between each generation of a profile";
+        return "show the closure difference between each version of a profile";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To show what changed between each generation of the NixOS system profile:",
-                "nix profile diff-closures --profile /nix/var/nix/profiles/system"
-            },
-        };
+        return
+          #include "profile-diff-closures.md"
+          ;
     }
 
     void run(ref<Store> store) override
@@ -429,7 +390,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
             if (prevGen) {
                 if (!first) std::cout << "\n";
                 first = false;
-                std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number);
+                std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number);
                 printClosureDiff(store,
                     store->followLinksToStorePath(prevGen->path),
                     store->followLinksToStorePath(gen.path),
@@ -458,6 +419,13 @@ struct CmdProfile : NixMultiCommand
         return "manage Nix profiles";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "profile.md"
+          ;
+    }
+
     void run() override
     {
         if (!command)
diff --git a/src/nix/profile.md b/src/nix/profile.md
new file mode 100644
index 000000000..d3ddcd3d1
--- /dev/null
+++ b/src/nix/profile.md
@@ -0,0 +1,107 @@
+R""(
+
+# Description
+
+`nix profile` allows you to create and manage *Nix profiles*. A Nix
+profile is a set of packages that can be installed and upgraded
+independently from each other. Nix profiles are versioned, allowing
+them to be rolled back easily.
+
+# Default profile
+
+The default profile used by `nix profile` is `$HOME/.nix-profile`,
+which, if it does not exist, is created as a symlink to
+`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the
+`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise.
+
+You can specify another profile location using `--profile` *path*.
+
+# Filesystem layout
+
+Profiles are versioned as follows. When using profile *path*, *path*
+is a symlink to *path*`-`*N*, where *N* is the current *version* of
+the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix
+store. For example:
+
+```console
+$ ls -l /nix/var/nix/profiles/per-user/alice/profile*
+lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link
+lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile
+lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile
+lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile
+```
+
+Each of these symlinks is a root for the Nix garbage collector.
+
+The contents of the store path corresponding to each version of the
+profile is a tree of symlinks to the files of the installed packages,
+e.g.
+
+```console
+$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/
+/nix/var/nix/profiles/per-user/eelco/profile-7-link/:
+total 20
+dr-xr-xr-x 2 root root 4096 Jan  1  1970 bin
+-r--r--r-- 2 root root 1402 Jan  1  1970 manifest.json
+dr-xr-xr-x 4 root root 4096 Jan  1  1970 share
+
+/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin:
+total 20
+lrwxrwxrwx 5 root root 79 Jan  1  1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
+lrwxrwxrwx 7 root root 87 Jan  1  1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
+lrwxrwxrwx 3 root root 79 Jan  1  1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
+
+/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications:
+total 12
+lrwxrwxrwx 4 root root 120 Jan  1  1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
+lrwxrwxrwx 7 root root 110 Jan  1  1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
+lrwxrwxrwx 3 root root 107 Jan  1  1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
+
+…
+```
+
+The file `manifest.json` records the provenance of the packages that
+are installed in this version of the profile. It looks like this:
+
+```json
+{
+  "version": 1,
+  "elements": [
+    {
+      "active": true,
+      "attrPath": "legacyPackages.x86_64-linux.zoom-us",
+      "originalUri": "flake:nixpkgs",
+      "storePaths": [
+        "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
+      ],
+      "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a"
+    },
+    …
+  ]
+}
+```
+
+Each object in the array `elements` denotes an installed package and
+has the following fields:
+
+* `originalUri`: The [flake reference](./nix3-flake.md) specified by
+  the user at the time of installation (e.g. `nixpkgs`). This is also
+  the flake reference that will be used by `nix profile upgrade`.
+
+* `uri`: The immutable flake reference to which `originalUri`
+  resolved.
+
+* `attrPath`: The flake output attribute that provided this
+  package. Note that this is not necessarily the attribute that the
+  user specified, but the one resulting from applying the default
+  attribute paths and prefixes; for instance, `hello` might resolve to
+  `packages.x86_64-linux.hello` and the empty string to
+  `defaultPackage.x86_64-linux`.
+
+* `storePath`: The paths in the Nix store containing the package.
+
+* `active`: Whether the profile contains symlinks to the files of this
+  package. If set to false, the package is kept in the Nix store, but
+  is not "visible" in the profile's symlink tree.
+
+)""

From 629af83b2d86d77305dc994b83f176a377106c3e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 14 Jul 2020 20:59:24 +0200
Subject: [PATCH 614/882] Provide a more meaningful error-message for
 `builtins.fetchGit` if a revision can't be checked out

A common pitfall when using e.g. `builtins.fetchGit` is the `fatal: not
a tree object`-error when trying to fetch a revision of a git-repository
that isn't on the `master` branch and no `ref` is specified.

In order to make clear what's the problem, I added a simple check
whether the revision in question exists and if it doesn't a more
meaningful error-message is displayed:

```
nix-repl> builtins.fetchGit { url = "https://github.com/owner/myrepo"; rev = "<commit not on master>"; }
moderror: --- Error -------------------------------------------------------------------- nix
Cannot find Git revision 'bf1cc5c648e6aed7360448a3745bb2fe4fbbf0e9' in ref 'master' of repository 'https://gitlab.com/Ma27/nvim.nix'! Please make sure that the rev exists on the ref you've specified or add allRefs = true; to fetchGit.
```

Closes #2431
---
 src/libfetchers/git.cc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index e7712c5fd..1f298c2d6 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -392,6 +392,28 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
+        RunOptions checkCommitOpts(
+            "git",
+            { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
+        );
+        checkCommitOpts.searchPath = true;
+        checkCommitOpts.mergeStderrToStdout = true;
+
+        auto result = runProgram(checkCommitOpts);
+        if (WEXITSTATUS(result.first) == 128
+            && result.second.find("bad file") != std::string::npos
+        ) {
+            throw Error(
+                "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
+                    "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
+                    ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
+                    "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
+                input.getRev()->gitRev(),
+                *input.getRef(),
+                actualUrl
+            );
+        }
+
         if (submodules) {
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);

From 2857b1baaf78bbadeec01adfae8a50fc0f2a254f Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 17 Jul 2020 20:34:57 +0200
Subject: [PATCH 615/882] Add explicit `allRefs = true;` argument to `fetchGit`

Sometimes it's necessary to fetch a git repository at a revision and
it's unknown which ref contains the revision in question. An example
would be a Cargo.lock which only provides the URL and the revision when
using a git repository as build input.

However it's considered a bad practice to perform a full checkout of a
repository since this may take a lot of time and can eat up a lot of
disk space. This patch makes a full checkout explicit by adding an
`allRefs` argument to `builtins.fetchGit` which fetches all refs if
explicitly set to true.

Closes #2409
---
 src/libfetchers/git.cc | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 1f298c2d6..81c647f89 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -59,12 +59,13 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
+        maybeGetBoolAttr(attrs, "allRefs");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
             if (std::regex_search(*ref, badGitRefRegex))
@@ -169,10 +170,12 @@ struct GitInputScheme : InputScheme
 
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+        bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
         if (submodules) cacheType += "-submodules";
+        if (allRefs) cacheType += "-all-refs";
 
         auto getImmutableAttrs = [&]()
         {
@@ -338,11 +341,15 @@ struct GitInputScheme : InputScheme
                     }
                 }
             } else {
-                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-                   git fetch to update the local ref to the remote ref. */
-                struct stat st;
-                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+                if (allRefs) {
+                    doFetch = true;
+                } else {
+                    /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                       git fetch to update the local ref to the remote ref. */
+                    struct stat st;
+                    doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+                }
             }
 
             if (doFetch) {
@@ -352,9 +359,11 @@ struct GitInputScheme : InputScheme
                 // we're using --quiet for now. Should process its stderr.
                 try {
                     auto ref = input.getRef();
-                    auto fetchRef = ref->compare(0, 5, "refs/") == 0
-                        ? *ref
-                        : "refs/heads/" + *ref;
+                    auto fetchRef = allRefs
+                        ? "refs/*"
+                        : ref->compare(0, 5, "refs/") == 0
+                            ? *ref
+                            : "refs/heads/" + *ref;
                     runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;

From 724b7f4fb660212a97ba6482208c299158720c5b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Dec 2020 11:15:29 +0100
Subject: [PATCH 616/882] Don't log from inside the logger

This deadlocks ProgressBar, e.g.

  # nix run --impure --no-substitute --store '/tmp/nix2?store=/foo' --expr 'derivation { builder = /nix/store/zi90rxslsm4mlr46l2xws1rm94g7pk8p-busybox-1.31.1-x86_64-unknown-linux-musl/bin/busybox; }'

leads to

  Thread 1 (Thread 0x7ffff6126e80 (LWP 12250)):
  #0  0x00007ffff7215d62 in __lll_lock_wait () from /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/lib/libpthread.so.0
  #1  0x00007ffff720e721 in pthread_mutex_lock () from /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/lib/libpthread.so.0
  #2  0x00007ffff7ad17fa in __gthread_mutex_lock (__mutex=0x6c5448) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/x86_64-unknown-linux-gnu/bits/gthr-default.h:749
  #3  std::mutex::lock (this=0x6c5448) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/std_mutex.h:100
  #4  std::unique_lock<std::mutex>::lock (this=0x7fffffff09a8, this=0x7fffffff09a8) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/unique_lock.h:141
  #5  std::unique_lock<std::mutex>::unique_lock (__m=..., this=0x7fffffff09a8) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/bits/unique_lock.h:71
  #6  nix::Sync<nix::ProgressBar::State, std::mutex>::Lock::Lock (s=0x6c5448, this=0x7fffffff09a0) at src/libutil/sync.hh:45
  #7  nix::Sync<nix::ProgressBar::State, std::mutex>::lock (this=0x6c5448) at src/libutil/sync.hh:85
  #8  nix::ProgressBar::logEI (this=0x6c5440, ei=...) at src/libmain/progress-bar.cc:131
  #9  0x00007ffff7608cfd in nix::Logger::logEI (ei=..., lvl=nix::lvlError, this=0x6c5440) at src/libutil/logging.hh:88
  #10 nix::getCodeLines (errPos=...) at src/libutil/error.cc:66
  #11 0x00007ffff76073f2 in nix::showErrorInfo (out=..., einfo=..., showTrace=<optimized out>) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/optional:897
  #12 0x00007ffff7ad19e7 in nix::ProgressBar::logEI (this=0x6c5440, ei=...) at src/libmain/progress-bar.cc:134
  #13 0x00007ffff7ab9d10 in nix::Logger::logEI (ei=..., lvl=nix::lvlError, this=0x6c5440) at src/libutil/logging.hh:88
  #14 nix::handleExceptions(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()>) (programName="/home/eelco/Dev/nix/outputs/out/bin/nix", fun=...) at src/libmain/shared.cc:328
  #15 0x000000000046226b in main (argc=<optimized out>, argv=<optimized out>) at /nix/store/h31cy7jm6g7cfqbhc5pm4rf9c53i3qfb-gcc-9.3.0/include/c++/9.3.0/ext/new_allocator.h:80
---
 src/libutil/error.cc | 50 +++++++++++++++++++-------------------------
 1 file changed, 21 insertions(+), 29 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 803a72953..e7dc3f1d3 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -62,35 +62,28 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
         LinesOfCode loc;
         try {
             AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
-            if (!fd) {
-                logError(SysError("opening file '%1%'", errPos.file).info());
-                return std::nullopt;
-            }
-            else
+            if (!fd) return {};
+
+            // count the newlines.
+            int count = 0;
+            string line;
+            int pl = errPos.line - 1;
+            do
             {
-                // count the newlines.
-                int count = 0;
-                string line;
-                int pl = errPos.line - 1;
-                do
-                {
-                    line = readLine(fd.get());
-                    ++count;
-                    if (count < pl)
-                    {
-                        ;
-                    }
-                    else if (count == pl) {
-                        loc.prevLineOfCode = line;
-                    } else if (count == pl + 1) {
-                        loc.errLineOfCode = line;
-                    } else if (count == pl + 2) {
-                        loc.nextLineOfCode = line;
-                        break;
-                    }
-                } while (true);
-                return loc;
-            }
+                line = readLine(fd.get());
+                ++count;
+                if (count < pl)
+                    ;
+                else if (count == pl)
+                    loc.prevLineOfCode = line;
+                else if (count == pl + 1)
+                    loc.errLineOfCode = line;
+                else if (count == pl + 2) {
+                    loc.nextLineOfCode = line;
+                    break;
+                }
+            } while (true);
+            return loc;
         }
         catch (EndOfFile & eof) {
             if (loc.errLineOfCode.has_value())
@@ -99,7 +92,6 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
                 return std::nullopt;
         }
         catch (std::exception & e) {
-            printError("error reading nix file: %s\n%s", errPos.file, e.what());
             return std::nullopt;
         }
     } else {

From e54971d019821dd213db028a80cd5fca85ee2ed6 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 8 Sep 2020 19:45:28 +0200
Subject: [PATCH 617/882] Document `allRefs` argument of `builtins.fetchTree`

---
 src/libexpr/primops/fetchTree.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 6e7ddde8e..133299030 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -324,6 +324,11 @@ static RegisterPrimOp primop_fetchGit({
           A Boolean parameter that specifies whether submodules should be
           checked out. Defaults to `false`.
 
+        - allRefs  
+          Whether to fetch all refs of the repository. With this argument being
+          true, it's possible to load a `rev` from *any* `ref` (by default only
+          `rev`s from the specified `ref` are supported).
+
       Here are some examples of how to use `fetchGit`.
 
         - To fetch a private repository over SSH:

From 897ae235fc2cef0ce711470a7b620241d82a1b09 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 22 Dec 2020 12:18:10 +0100
Subject: [PATCH 618/882] tests/fetchGit: test behavior of `allRefs = true;`

---
 tests/fetchGit.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 76390fa59..1e8963d76 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -41,6 +41,19 @@ export _NIX_FORCE_HTTP=1
 path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
 
+# Fetch a rev from another branch
+git -C $repo checkout -b devtest
+echo "different file" >> $TEST_ROOT/git/differentbranch
+git -C $repo add differentbranch
+git -C $repo commit -m 'Test2'
+git -C $repo checkout master
+devrev=$(git -C $repo rev-parse devtest)
+out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$?
+[[ $status == 1 ]]
+[[ $out =~ 'Cannot find Git revision' ]]
+
+[[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]]
+
 # In pure eval mode, fetchGit without a revision should fail.
 [[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") = world ]]
 (! nix eval --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")")

From 5373f4be3b1427ed73303448a1fa801726f4dfa0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Dec 2020 12:28:50 +0100
Subject: [PATCH 619/882] chrootHelper: Handle symlinks in the root directory

This is necessary on Ubuntu where /bin and /lib* are symlinks.
---
 src/nix/run.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 92a52c6cd..ec61fc79a 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -258,14 +258,16 @@ void chrootHelper(int argc, char * * argv)
 
         for (auto entry : readDirectory("/")) {
             auto src = "/" + entry.name;
-            auto st = lstat(src);
-            if (!S_ISDIR(st.st_mode)) continue;
             Path dst = tmpDir + "/" + entry.name;
             if (pathExists(dst)) continue;
-            if (mkdir(dst.c_str(), 0700) == -1)
-                throw SysError("creating directory '%s'", dst);
-            if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                throw SysError("mounting '%s' on '%s'", src, dst);
+            auto st = lstat(src);
+            if (S_ISDIR(st.st_mode)) {
+                if (mkdir(dst.c_str(), 0700) == -1)
+                    throw SysError("creating directory '%s'", dst);
+                if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+                    throw SysError("mounting '%s' on '%s'", src, dst);
+            } else if (S_ISLNK(st.st_mode))
+                createSymlink(readLink(src), dst);
         }
 
         char * cwd = getcwd(0, 0);

From 75efa421340b8fb2be6cf5351d3ef36a93b294e1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Dec 2020 14:43:20 +0100
Subject: [PATCH 620/882] Move <nix/fetchurl.nix> into the nix binary

This makes the statically linked nix binary just work, without needing
any additional files.
---
 Makefile                               |  1 -
 corepkgs/local.mk                      |  4 ----
 src/libexpr/eval.cc                    |  7 ++-----
 src/libexpr/eval.hh                    |  2 ++
 {corepkgs => src/libexpr}/fetchurl.nix |  0
 src/libexpr/local.mk                   |  2 +-
 src/libexpr/parser.y                   |  4 ++++
 src/libexpr/primops.cc                 | 10 +++++++++-
 tests/fetchurl.sh                      | 14 +++++++-------
 tests/lang/eval-okay-search-path.nix   |  5 ++---
 10 files changed, 27 insertions(+), 22 deletions(-)
 delete mode 100644 corepkgs/local.mk
 rename {corepkgs => src/libexpr}/fetchurl.nix (100%)

diff --git a/Makefile b/Makefile
index c50d2c40f..f80b8bb82 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,6 @@ makefiles = \
   src/nix/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
-  corepkgs/local.mk \
   misc/bash/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
diff --git a/corepkgs/local.mk b/corepkgs/local.mk
deleted file mode 100644
index 0bc91cfab..000000000
--- a/corepkgs/local.mk
+++ /dev/null
@@ -1,4 +0,0 @@
-corepkgs_FILES = \
-  fetchurl.nix
-
-$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 5a641d02c..ead5bf8c7 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -402,11 +402,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
 
-    try {
-        addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
-    } catch (Error &) {
-    }
-
     if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
 
@@ -457,6 +452,8 @@ Path EvalState::checkSourcePath(const Path & path_)
      */
     Path abspath = canonPath(path_);
 
+    if (hasPrefix(abspath, corepkgsPrefix)) return abspath;
+
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(abspath, i)) {
             found = true;
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 0e1f61baa..e3eaed6d3 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -432,4 +432,6 @@ struct EvalSettings : Config
 
 extern EvalSettings evalSettings;
 
+static const std::string corepkgsPrefix{"/__corepkgs__/"};
+
 }
diff --git a/corepkgs/fetchurl.nix b/src/libexpr/fetchurl.nix
similarity index 100%
rename from corepkgs/fetchurl.nix
rename to src/libexpr/fetchurl.nix
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 519da33f7..26c53d301 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -40,6 +40,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index a4c84c526..85eb05d61 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -698,6 +698,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
+
+    if (hasPrefix(path, "nix/"))
+        return corepkgsPrefix + path.substr(4);
+
     throw ThrownError({
         .hint = hintfmt(evalSettings.pureEval
             ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 18438f681..c73a94f4e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -164,7 +164,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         state.forceFunction(**fun, pos);
         mkApp(v, **fun, w);
         state.forceAttrs(v, pos);
-    } else {
+    }
+
+    else if (path == corepkgsPrefix + "fetchurl.nix") {
+        state.eval(state.parseExprFromString(
+            #include "fetchurl.nix.gen.hh"
+            , "/"), v);
+    }
+
+    else {
         if (!vScope)
             state.evalFile(realPath, v);
         else {
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 10ec0173a..cd84e9a4c 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
 # Test fetching a flat file.
 hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
+outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -14,7 +14,7 @@ clearStore
 
 hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
+outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -25,7 +25,7 @@ hash=$(nix hash file ./fetchurl.sh)
 
 [[ $hash =~ ^sha256- ]]
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
+outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -38,10 +38,10 @@ hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
 
 storePath=$(nix --store $other_store store add-file ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
+outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
-nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
+nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
           --no-out-link --substituters $other_store
 
 # Test unpacking a NAR.
@@ -55,7 +55,7 @@ nix-store --dump $TEST_ROOT/archive > $nar
 
 hash=$(nix-hash --flat --type sha256 $nar)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \
+outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \
           --arg unpack true --argstr name xyzzy --no-out-link)
 
 echo $outPath | grep -q 'xyzzy'
@@ -69,7 +69,7 @@ nix-store --delete $outPath
 narxz=$TEST_ROOT/archive.nar.xz
 rm -f $narxz
 xz --keep $nar
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$narxz --argstr sha256 $hash \
+outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$narxz --argstr sha256 $hash \
           --arg unpack true --argstr name xyzzy --no-out-link)
 
 test -x $outPath/fetchurl.sh
diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix
index c5a123d04..6fe33decc 100644
--- a/tests/lang/eval-okay-search-path.nix
+++ b/tests/lang/eval-okay-search-path.nix
@@ -1,10 +1,9 @@
 with import ./lib.nix;
 with builtins;
 
-assert pathExists <nix/fetchurl.nix>;
+assert isFunction (import <nix/fetchurl.nix>);
 
-assert length __nixPath == 6;
-assert length (filter (x: x.prefix == "nix") __nixPath) == 1;
+assert length __nixPath == 5;
 assert length (filter (x: baseNameOf x.path == "dir4") __nixPath) == 1;
 
 import <a.nix> + import <b.nix> + import <c.nix> + import <dir5/c.nix>

From e27044216bf597710893e0366dbff60efbdaf0a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Dec 2020 16:23:57 +0100
Subject: [PATCH 621/882] Fix tests

---
 src/libutil/tests/logging.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 7e53f17c6..5b32c84a4 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -49,7 +49,7 @@ namespace nix {
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
+        ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
     }
 
     TEST(logEI, appendingHintsToPreviousError) {
@@ -208,7 +208,7 @@ namespace nix {
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
     TEST(logError, logErrorWithOnlyHintAndName) {
@@ -290,7 +290,7 @@ namespace nix {
         logError(e.info());
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
     }
 
     TEST(addTrace, hideTracesWithoutShowTrace) {

From c9279b831e91a762851464826eaa8a8e30979578 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Dec 2020 13:19:53 +0100
Subject: [PATCH 622/882] Add 'nix flake' manpages

---
 src/nix/flake-archive.md     |  29 ++
 src/nix/flake-check.md       |  68 +++++
 src/nix/flake-clone.md       |  18 ++
 src/nix/flake-info.md        |  99 ++++++
 src/nix/flake-init.md        |  54 ++++
 src/nix/flake-list-inputs.md |  23 ++
 src/nix/flake-new.md         |  34 +++
 src/nix/flake-show.md        |  38 +++
 src/nix/flake-update.md      |  53 ++++
 src/nix/flake.cc             |  94 ++++--
 src/nix/flake.md             | 566 +++++++++++++++++++++++++++++++++++
 11 files changed, 1046 insertions(+), 30 deletions(-)
 create mode 100644 src/nix/flake-archive.md
 create mode 100644 src/nix/flake-check.md
 create mode 100644 src/nix/flake-clone.md
 create mode 100644 src/nix/flake-info.md
 create mode 100644 src/nix/flake-init.md
 create mode 100644 src/nix/flake-list-inputs.md
 create mode 100644 src/nix/flake-new.md
 create mode 100644 src/nix/flake-show.md
 create mode 100644 src/nix/flake-update.md
 create mode 100644 src/nix/flake.md

diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md
new file mode 100644
index 000000000..85bbeeb16
--- /dev/null
+++ b/src/nix/flake-archive.md
@@ -0,0 +1,29 @@
+R""(
+
+# Examples
+
+* Copy the `dwarffs` flake and its dependencies to a binary cache:
+
+  ```console
+  # nix flake archive --to file:///tmp/my-cache dwarffs
+  ```
+
+* Fetch the `dwarffs` flake and its dependencies to the local Nix
+  store:
+
+  ```console
+  # nix flake archive dwarffs
+  ```
+
+* Print the store paths of the flake sources of NixOps without
+  fetching them:
+
+  ```console
+  # nix flake archive --json --dry-run nixops
+  ```
+
+# Description
+
+FIXME
+
+)""
diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md
new file mode 100644
index 000000000..dc079ba0c
--- /dev/null
+++ b/src/nix/flake-check.md
@@ -0,0 +1,68 @@
+R""(
+
+# Examples
+
+* Evaluate the flake in the current directory, and build its checks:
+
+  ```console
+  # nix flake check
+  ```
+
+* Verify that the `patchelf` flake evaluates, but don't build its
+  checks:
+
+  ```console
+  # nix flake check --no-build github:NixOS/patchelf
+  ```
+
+# Description
+
+This command verifies that the flake specified by flake reference
+*flake-url* can be evaluated successfully (as detailed below), and
+that the derivations specified by the flake's `checks` output can be
+built successfully.
+
+# Evaluation checks
+
+This following flake output attributes must be derivations:
+
+* `checks.`*system*`.`*name*
+* `defaultPackage.`*system*`
+* `devShell.`*system*`
+* `nixosConfigurations.`*name*`.config.system.build.toplevel
+* `packages.`*system*`.`*name*
+
+The following flake output attributes must be [app
+definitions](./nix3-run.md):
+
+* `apps.`*system*`.`*name*
+* `defaultApp.`*system*`
+
+The following flake output attributes must be [template
+definitions](./nix3-flake-init.md):
+
+* `defaultTemplate`
+* `templates`.`*name*
+
+The following flake output attributes must be *Nixpkgs overlays*:
+
+* `overlay`
+* `overlays`.`*name*
+
+The following flake output attributes must be *NixOS modules*:
+
+* `nixosModule`
+* `nixosModules`.`*name*
+
+The following flake output attributes must be
+[bundlers](./nix3-bundle.md):
+
+* `bundlers`.`*name*
+* `defaultBundler`
+
+In addition, the `hydraJobs` output is evaluated in the same way as
+Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested
+attribute set of derivations). Similarly, the
+`legacyPackages`.*system* output is evaluated like `nix-env -qa`.
+
+)""
diff --git a/src/nix/flake-clone.md b/src/nix/flake-clone.md
new file mode 100644
index 000000000..36cb96051
--- /dev/null
+++ b/src/nix/flake-clone.md
@@ -0,0 +1,18 @@
+R""(
+
+# Examples
+
+* Check out the source code of the `dwarffs` flake and build it:
+
+  ```console
+  # nix flake clone dwarffs --dest dwarffs
+  # cd dwarffs
+  # nix build
+  ```
+
+# Description
+
+This command performs a Git or Mercurial clone of the repository
+containing the source code of the flake *flake-url*.
+
+)""
diff --git a/src/nix/flake-info.md b/src/nix/flake-info.md
new file mode 100644
index 000000000..fda3171db
--- /dev/null
+++ b/src/nix/flake-info.md
@@ -0,0 +1,99 @@
+R""(
+
+# Examples
+
+* Show what `nixpkgs` resolves to:
+
+  ```console
+  # nix flake info nixpkgs
+  Resolved URL:  github:NixOS/nixpkgs
+  Locked URL:    github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4
+  Description:   A collection of packages for the Nix package manager
+  Path:          /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source
+  Revision:      b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4
+  Last modified: 2020-12-23 12:36:12
+  ```
+
+* Show information about `dwarffs` in JSON format:
+
+  ```console
+  # nix flake info dwarffs --json | jq .
+  {
+    "description": "A filesystem that fetches DWARF debug info from the Internet on demand",
+    "lastModified": 1597153508,
+    "locked": {
+      "lastModified": 1597153508,
+      "narHash": "sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=",
+      "owner": "edolstra",
+      "repo": "dwarffs",
+      "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5",
+      "type": "github"
+    },
+    "original": {
+      "id": "dwarffs",
+      "type": "indirect"
+    },
+    "originalUrl": "flake:dwarffs",
+    "path": "/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source",
+    "resolved": {
+      "owner": "edolstra",
+      "repo": "dwarffs",
+      "type": "github"
+    },
+    "resolvedUrl": "github:edolstra/dwarffs",
+    "revision": "d181d714fd36eb06f4992a1997cd5601e26db8f5",
+    "url": "github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5"
+  }
+  ```
+
+# Description
+
+This command shows information about the flake specified by the flake
+reference *flake-url*. It resolves the flake reference using the
+[flake registry](./nix3-registry.md), fetches it, and prints some meta
+data. This includes:
+
+* `Resolved URL`: If *flake-url* is a flake identifier, then this is
+  the flake reference that specifies its actual location, looked up in
+  the flake registry.
+
+* `Locked URL`: A flake reference that contains a commit or content
+  hash and thus uniquely identifies a specific flake version.
+
+* `Description`: A one-line description of the flake, taken from the
+  `description` field in `flake.nix`.
+
+* `Path`: The store path containing the source code of the flake.
+
+* `Revision`: The Git or Mercurial commit hash of the locked flake.
+
+* `Revisions`: The number of ancestors of the Git or Mercurial commit
+  of the locked flake. Note that this is not available for `github`
+  flakes.
+
+* `Last modified`: For Git or Mercurial flakes, this is the commit
+  time of the commit of the locked flake; for tarball flakes, it's the
+  most recent timestamp of any file inside the tarball.
+
+With `--json`, the output is a JSON object with the following fields:
+
+* `original` and `originalUrl`: The flake reference specified by the
+  user (*flake-url*) in attribute set and URL representation.
+
+* `resolved` and `resolvedUrl`: The resolved flake reference (see
+  above) in attribute set and URL representation.
+
+* `locked` and `lockedUrl`: The locked flake reference (see above) in
+  attribute set and URL representation.
+
+* `description`: See `Description` above.
+
+* `path`: See `Path` above.
+
+* `revision`: See `Revision` above.
+
+* `revCount`: See `Revisions` above.
+
+* `lastModified`: See `Last modified` above.
+
+)""
diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md
new file mode 100644
index 000000000..c66154ad5
--- /dev/null
+++ b/src/nix/flake-init.md
@@ -0,0 +1,54 @@
+R""(
+
+# Examples
+
+* Create a flake using the default template:
+
+  ```console
+  # nix flake init
+  ```
+
+* List available templates:
+
+  ```console
+  # nix flake show templates
+  ```
+
+* Create a flake from a specific template:
+
+  ```console
+  # nix flake init -t templates#simpleContainer
+  ```
+
+# Description
+
+This command creates a flake in the current directory by copying the
+files of a template. It will not overwrite existing files. The default
+template is `templates#defaultTemplate`, but this can be overriden
+using `-t`.
+
+# Template definitions
+
+A flake can declare templates through its `templates` and
+`defaultTemplate` output attributes. A template has two attributes:
+
+* `description`: A one-line description of the template, in CommonMark
+  syntax.
+
+* `path`: The path of the directory to be copied.
+
+Here is an example:
+
+```
+outputs = { self }: {
+
+  templates.rust = {
+    path = ./rust;
+    description = "A simple Rust/Cargo project";
+  };
+
+  templates.defaultTemplate = self.templates.rust;
+}
+```
+
+)""
diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md
new file mode 100644
index 000000000..250e13be0
--- /dev/null
+++ b/src/nix/flake-list-inputs.md
@@ -0,0 +1,23 @@
+R""(
+
+# Examples
+
+* Show the inputs of the `hydra` flake:
+
+  ```console
+  # nix flake list-inputs github:NixOS/hydra
+  github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7
+  ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67
+  │   ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f
+  │   └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31
+  └───nixpkgs follows input 'nix/nixpkgs'
+  ```
+
+# Description
+
+This command shows the inputs of the flake specified by the flake
+referenced *flake-url*. Since it prints the locked inputs that result
+from generating or updating the lock file, this command essentially
+displays the contents of the flake's lock file in human-readable form.
+
+)""
diff --git a/src/nix/flake-new.md b/src/nix/flake-new.md
new file mode 100644
index 000000000..725695c01
--- /dev/null
+++ b/src/nix/flake-new.md
@@ -0,0 +1,34 @@
+R""(
+
+# Examples
+
+* Create a flake using the default template in the directory `hello`:
+
+  ```console
+  # nix flake new hello
+  ```
+
+* List available templates:
+
+  ```console
+  # nix flake show templates
+  ```
+
+* Create a flake from a specific template in the directory `hello`:
+
+  ```console
+  # nix flake new hello -t templates#trivial
+  ```
+
+# Description
+
+This command creates a flake in the directory `dest-dir`, which must
+not already exist. It's equivalent to:
+
+```console
+# mkdir dest-dir
+# cd dest-dir
+# nix flake init
+```
+
+)""
diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md
new file mode 100644
index 000000000..1a42c44a0
--- /dev/null
+++ b/src/nix/flake-show.md
@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Show the output attributes provided by the `patchelf` flake:
+
+  ```console
+  github:NixOS/patchelf/f34751b88bd07d7f44f5cd3200fb4122bf916c7e
+  ├───checks
+  │   ├───aarch64-linux
+  │   │   └───build: derivation 'patchelf-0.12.20201207.f34751b'
+  │   ├───i686-linux
+  │   │   └───build: derivation 'patchelf-0.12.20201207.f34751b'
+  │   └───x86_64-linux
+  │       └───build: derivation 'patchelf-0.12.20201207.f34751b'
+  ├───defaultPackage
+  │   ├───aarch64-linux: package 'patchelf-0.12.20201207.f34751b'
+  │   ├───i686-linux: package 'patchelf-0.12.20201207.f34751b'
+  │   └───x86_64-linux: package 'patchelf-0.12.20201207.f34751b'
+  ├───hydraJobs
+  │   ├───build
+  │   │   ├───aarch64-linux: derivation 'patchelf-0.12.20201207.f34751b'
+  │   │   ├───i686-linux: derivation 'patchelf-0.12.20201207.f34751b'
+  │   │   └───x86_64-linux: derivation 'patchelf-0.12.20201207.f34751b'
+  │   ├───coverage: derivation 'patchelf-coverage-0.12.20201207.f34751b'
+  │   ├───release: derivation 'patchelf-0.12.20201207.f34751b'
+  │   └───tarball: derivation 'patchelf-tarball-0.12.20201207.f34751b'
+  └───overlay: Nixpkgs overlay
+  ```
+
+# Description
+
+This command shows the output attributes provided by the flake
+specified by flake reference *flake-url*. These are the top-level
+attributes in the `outputs` of the flake, as well as lower-level
+attributes for some standard outputs (e.g. `packages` or `checks`).
+
+)""
diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md
new file mode 100644
index 000000000..a2ffedd2a
--- /dev/null
+++ b/src/nix/flake-update.md
@@ -0,0 +1,53 @@
+R""(
+
+# Examples
+
+* Update the `nixpkgs` and `nix` inputs of the flake in the current
+  directory:
+
+  ```console
+  # nix flake update --update-input nixpkgs --update-input nix
+  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  ```
+
+* Recreate the lock file (i.e. update all inputs) and commit the new
+  lock file:
+
+  ```console
+  # nix flake update --recreate-lock-file --commit-lock-file
+  …
+  warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
+  ```
+
+# Description
+
+This command updates the lock file of a flake (`flake.lock`) so that
+it contains a lock for every flake input specified in
+`flake.nix`. Note that every command that operates on a flake will
+also update the lock file if needed, and supports the same
+flags. Therefore,
+
+```console
+# nix flake update --update-input nixpkgs
+# nix build
+```
+
+is equivalent to:
+
+```console
+# nix build --update-input nixpkgs
+```
+
+Thus, this command is only useful if you want to update the lock file
+separately from any other action such as building.
+
+> **Note**
+>
+> This command does *not* update locks that are already present unless
+> you explicitly ask for it using `--update-input` or
+> `--recreate-lock-file`. Thus, if the lock file already has locks for
+> every input, then `nix flake update` (without arguments) does
+> nothing.
+
+)""
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 066430c5d..2b91faa64 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -104,6 +104,13 @@ struct CmdFlakeUpdate : FlakeCommand
         return "update flake lock file";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-update.md"
+          ;
+    }
+
     void run(nix::ref<nix::Store> store) override
     {
         /* Use --refresh by default for 'nix flake update'. */
@@ -134,6 +141,13 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
         return "list info about a given flake";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-info.md"
+          ;
+    }
+
     void run(nix::ref<nix::Store> store) override
     {
         auto flake = getFlake();
@@ -153,6 +167,13 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
         return "list flake inputs";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-list-inputs.md"
+          ;
+    }
+
     void run(nix::ref<nix::Store> store) override
     {
         auto flake = lockFlake();
@@ -211,6 +232,13 @@ struct CmdFlakeCheck : FlakeCommand
         return "check whether the flake evaluates and run its tests";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-check.md"
+          ;
+    }
+
     void run(nix::ref<nix::Store> store) override
     {
         settings.readOnlyMode = !build;
@@ -631,22 +659,11 @@ struct CmdFlakeInit : CmdFlakeInitCommon
         return "create a flake in the current directory from a template";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To create a flake using the default template:",
-                "nix flake init"
-            },
-            Example{
-                "To see available templates:",
-                "nix flake show templates"
-            },
-            Example{
-                "To create a flake from a specific template:",
-                "nix flake init -t templates#nixos-container"
-            },
-        };
+        return
+          #include "flake-init.md"
+          ;
     }
 
     CmdFlakeInit()
@@ -662,6 +679,13 @@ struct CmdFlakeNew : CmdFlakeInitCommon
         return "create a flake in the specified directory from a template";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-new.md"
+          ;
+    }
+
     CmdFlakeNew()
     {
         expectArgs({
@@ -681,6 +705,13 @@ struct CmdFlakeClone : FlakeCommand
         return "clone flake repository";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-clone.md"
+          ;
+    }
+
     CmdFlakeClone()
     {
         addFlag({
@@ -720,22 +751,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
         return "copy a flake and all its inputs to a store";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
-            Example{
-                "To copy the dwarffs flake and its dependencies to a binary cache:",
-                "nix flake archive --to file:///tmp/my-cache dwarffs"
-            },
-            Example{
-                "To fetch the dwarffs flake and its dependencies to the local Nix store:",
-                "nix flake archive dwarffs"
-            },
-            Example{
-                "To print the store paths of the flake sources of NixOps without fetching them:",
-                "nix flake archive --json --dry-run nixops"
-            },
-        };
+        return
+          #include "flake-archive.md"
+          ;
     }
 
     void run(nix::ref<nix::Store> store) override
@@ -797,6 +817,13 @@ struct CmdFlakeShow : FlakeCommand
         return "show the outputs provided by a flake";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake-show.md"
+          ;
+    }
+
     void run(nix::ref<nix::Store> store) override
     {
         auto state = getEvalState();
@@ -955,6 +982,13 @@ struct CmdFlake : NixMultiCommand
         return "manage Nix flakes";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "flake.md"
+          ;
+    }
+
     void run() override
     {
         if (!command)
diff --git a/src/nix/flake.md b/src/nix/flake.md
new file mode 100644
index 000000000..440c45dd1
--- /dev/null
+++ b/src/nix/flake.md
@@ -0,0 +1,566 @@
+R""(
+
+# Description
+
+`nix flake` provides subcommands for creating, modifying and querying
+*Nix flakes*. Flakes are the unit for packaging Nix code in a
+reproducible and discoverable way. They can have dependencies on other
+flakes, making it possible to have multi-repository Nix projects.
+
+A flake is a filesystem tree (typically fetched from a Git repository
+or a tarball) that contains a file named `flake.nix` in the root
+directory. `flake.nix` specifies some metadata about the flake such as
+dependencies (called *inputs*), as well as its *outputs* (the Nix
+values such as packages or NixOS modules provided by the flake).
+
+# Flake references
+
+Flake references (*flakerefs*) are a way to specify the location of a
+flake. These have two different forms:
+
+* An attribute set representation, e.g.
+
+  ```nix
+  {
+    type = "github";
+    owner = "NixOS";
+    repo = "nixpkgs";
+  }
+  ```
+
+  The only required attribute is `type`. The supported types are
+  listed below.
+
+* A URL-like syntax, e.g.
+
+  ```
+  github:NixOS/nixpkgs
+  ```
+
+  These are used on the command line as a more convenient alternative
+  to the attribute set representation. For instance, in the command
+
+  ```console
+  # nix build github:NixOS/nixpkgs#hello
+  ```
+
+  `github:NixOS/nixpkgs` is a flake reference (while `hello` is an
+  output attribute). They are also allowed in the `inputs` attribute
+  of a flake, e.g.
+
+  ```nix
+  inputs.nixpkgs.url = github:NixOS/nixpkgs;
+  ```
+
+  is equivalent to
+
+  ```nix
+  inputs.nixpkgs = {
+    type = "github";
+    owner = "NixOS";
+    repo = "nixpkgs";
+  };
+  ```
+
+## Examples
+
+Here are some examples of flake references in their URL-like representation:
+
+* `.`: The flake in the current directory.
+* `/home/alice/src/patchelf`: A flake in some other directory.
+* `nixpkgs`: The `nixpkgs` entry in the flake registry.
+* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs`
+  entry in the flake registry, with its Git revision overriden to a
+  specific value.
+* `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs`
+  repository on GitHub.
+* `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the
+  `nixpkgs` repository.
+* `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A
+  specific revision of the `nixpkgs` repository.
+* `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory
+  of a GitHub repository.
+* `git+https://github.com/NixOS/patchelf`: A Git repository.
+* `git+https://github.com/NixOS/patchelf?ref=master`: A specific
+  branch of a Git repository.
+* `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`:
+  A specific branch *and* revision of a Git repository.
+* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball
+  flake.
+
+## Flake reference attributes
+
+The following generic flake reference attributes are supported:
+
+* `dir`: The subdirectory of the flake in which `flake.nix` is
+  located. This parameter enables having multiple flakes in a
+  repository or tarball. The default is the root directory of the
+  flake.
+
+* `narHash`: The hash of the NAR serialisation (in SRI format) of the
+  contents of the flake. This is useful for flake types such as
+  tarballs that lack a unique content identifier such as a Git commit
+  hash.
+
+In addition, the following attributes are common to several flake
+reference types:
+
+* `rev`: A Git or Mercurial commit hash.
+
+* `ref`: A Git or Mercurial branch or tag name.
+
+Finally, some attribute are typically not specified by the user, but
+can occur in *locked* flake references and are available to Nix code:
+
+* `revCount`: The number of ancestors of the commit `rev`.
+
+* `lastModified`: The timestamp (in seconds since the Unix epoch) of
+  the last modification of this version of the flake. For
+  Git/Mercurial flakes, this is the commit time of commit *rev*, while
+  for tarball flakes, it's the most recent timestamp of any file
+  inside the tarball.
+
+## Types
+
+Currently the `type` attribute can be one of the following:
+
+* `path`: arbitrary local directories, or local Git trees. The
+  required attribute `path` specifies the path of the flake. The URL
+  form is
+
+  ```
+  [path:]<path>(\?<params)?
+  ```
+
+  where *path* is an absolute path.
+
+  *path* must be a directory in the file system containing a file
+  named `flake.nix`.
+
+  If the directory or any of its parents is a Git repository, then
+  this is essentially equivalent to `git+file://<path>` (see below),
+  except that the `dir` parameter is derived automatically. For
+  example, if `/foo/bar` is a Git repository, then the flake reference
+  `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`.
+
+  If the directory is not inside a Git repository, then the flake
+  contents is the entire contents of *path*.
+
+  *path* generally must be an absolute path. However, on the command
+  line, it can be a relative path (e.g. `.` or `./foo`) which is
+  interpreted as relative to the current directory. In this case, it
+  must start with `.` to avoid ambiguity with registry lookups
+  (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative
+  path).
+
+* `git`: Git repositories. The location of the repository is specified
+  by the attribute `url`.
+
+  They have the URL form
+
+  ```
+  git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
+  ```
+
+  The `ref` attribute defaults to `master`.
+
+  The `rev` attribute must denote a commit that exists in the branch
+  or tag specified by the `ref` attribute, since Nix doesn't do a full
+  clone of the remote repository by default (and the Git protocol
+  doesn't allow fetching a `rev` without a known `ref`). The default
+  is the commit currently pointed to by `ref`.
+
+  For example, the following are valid Git flake references:
+
+  * `git+https://example.org/my/repo`
+  * `git+https://example.org/my/repo?dir=flake1`
+  * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3`
+  * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4`
+  * `git+file:///home/my-user/some-repo/some-repo`
+
+* `mercurial`: Mercurial repositories. The URL form is similar to the
+  `git` type, except that the URL schema must be one of `hg+http`,
+  `hg+https`, `hg+ssh` or `hg+file`.
+
+* `tarball`: Tarballs. The location of the tarball is specified by the
+  attribute `url`.
+
+  In URL form, the schema must be `http://`, `https://` or `file://`
+  URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`
+  or `.tar.bz2`.
+
+* `github`: A more efficient way to fetch repositories from
+  GitHub. The following attributes are required:
+
+  * `owner`: The owner of the repository.
+
+  * `repo`: The name of the repository.
+
+  These are downloaded as tarball archives, rather than
+  through Git. This is often much faster and uses less disk space
+  since it doesn't require fetching the entire history of the
+  repository. On the other hand, it doesn't allow incremental fetching
+  (but full downloads are often faster than incremental fetches!).
+
+  The URL syntax for `github` flakes is:
+
+  ```
+  github:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?
+  ```
+
+  `<rev-or-ref>` specifies the name of a branch or tag (`ref`), or a
+  commit hash (`rev`). Note that unlike Git, GitHub allows fetching by
+  commit hash without specifying a branch or tag.
+
+  Some examples:
+
+  * `github:edolstra/dwarffs`
+  * `github:edolstra/dwarffs/unstable`
+  * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31`
+
+* `indirect`: Indirections through the flake registry. These have the
+  form
+
+  ```
+  [flake:]<flake-id>(/<rev-or-ref>(/rev)?)?
+  ```
+
+  These perform a lookup of `<flake-id>` in the flake registry. or
+  example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake
+  references. The specified `rev` and/or `ref` are merged with the
+  entry in the registry; see [nix registry](./nix3-registry.md) for
+  details.
+
+# Flake format
+
+As an example, here is a simple `flake.nix` that depends on the
+Nixpkgs flake and provides a single package (i.e. an installable
+derivation):
+
+```nix
+{
+  description = "A flake for building Hello World";
+
+  inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03;
+
+  outputs = { self, nixpkgs }: {
+
+    defaultPackage.x86_64-linux =
+      # Notice the reference to nixpkgs here.
+      with import nixpkgs { system = "x86_64-linux"; };
+      stdenv.mkDerivation {
+        name = "hello";
+        src = self;
+        buildPhase = "gcc -o hello ./hello.c";
+        installPhase = "mkdir -p $out/bin; install -t $out/bin hello";
+      };
+
+  };
+}
+```
+
+The following attributes are supported in `flake.nix`:
+
+* `description`: A short, one-line description of the flake.
+
+* `inputs`: An attrset specifying the dependencies of the flake
+  (described below).
+
+* `outputs`: A function that, given an attribute set containing the
+  outputs of each of the input flakes keyed by their identifier,
+  yields the Nix values provided by this flake. Thus, in the example
+  above, `inputs.nixpkgs` contains the result of the call to the
+  `outputs` function of the `nixpkgs` flake.
+
+  In addition to the outputs of each input, each input in `inputs`
+  also contains some metadata about the inputs. These are:
+
+  * `outPath`: The path in the Nix store of the flake's source tree.
+
+  * `rev`: The commit hash of the flake's repository, if applicable.
+
+  * `revCount`: The number of ancestors of the revision `rev`. This is
+    not available for `github` repositories, since they're fetched as
+    tarballs rather than as Git repositories.
+
+  * `lastModifiedDate`: The commit time of the revision `rev`, in the
+    format `%Y%m%d%H%M%S` (e.g. `20181231100934`). Unlike `revCount`,
+    this is available for both Git and GitHub repositories, so it's
+    useful for generating (hopefully) monotonically increasing version
+    strings.
+
+  * `lastModified`: The commit time of the revision `rev` as an integer
+    denoting the number of seconds since 1970.
+
+  * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of
+    the flake's source tree.
+
+  The value returned by the `outputs` function must be an attribute
+  set. The attributes can have arbitrary values; however, various
+  `nix` subcommands require specific attributes to have a specific
+  value (e.g. `packages.x86_64-linux` must be an attribute set of
+  derivations built for the `x86_64-linux` platform).
+
+## Flake inputs
+
+The attribute `inputs` specifies the dependencies of a flake, as an
+attrset mapping input names to flake references. For example, the
+following specifies a dependency on the `nixpkgs` and `import-cargo`
+repositories:
+
+```nix
+# A GitHub repository.
+inputs.import-cargo = {
+  type = "github";
+  owner = "edolstra";
+  repo = "import-cargo";
+};
+
+# An indirection through the flake registry.
+inputs.nixpkgs = {
+  type = "indirect";
+  id = "nixpkgs";
+};
+```
+
+Alternatively, you can use the URL-like syntax:
+
+```nix
+inputs.import-cargo.url = github:edolstra/import-cargo;
+inputs.nixpkgs.url = "nixpkgs";
+```
+
+Each input is fetched, evaluated and passed to the `outputs` function
+as a set of attributes with the same name as the corresponding
+input. The special input named `self` refers to the outputs and source
+tree of *this* flake. Thus, a typical `outputs` function looks like
+this:
+
+```nix
+outputs = { self, nixpkgs, import-cargo }: {
+  ... outputs ...
+};
+```
+
+It is also possible to omit an input entirely and *only* list it as
+expected function argument to `outputs`. Thus,
+
+```nix
+outputs = { self, nixpkgs }: ...;
+```
+
+without an `inputs.nixpkgs` attribute is equivalent to
+
+```nix
+inputs.nixpkgs = {
+  type = "indirect";
+  id = "nixpkgs";
+};
+```
+
+Repositories that don't contain a `flake.nix` can also be used as
+inputs, by setting the input's `flake` attribute to `false`:
+
+```nix
+inputs.grcov = {
+  type = "github";
+  owner = "mozilla";
+  repo = "grcov";
+  flake = false;
+};
+
+outputs = { self, nixpkgs, grcov }: {
+  packages.x86_64-linux.grcov = stdenv.mkDerivation {
+    src = grcov;
+    ...
+  };
+};
+```
+
+Transitive inputs can be overriden from a `flake.nix` file. For
+example, the following overrides the `nixpkgs` input of the `nixops`
+input:
+
+```nix
+inputs.nixops.inputs.nixpkgs = {
+  type = "github";
+  owner = "my-org";
+  repo = "nixpkgs";
+};
+```
+
+It is also possible to "inherit" an input from another input. This is
+useful to minimize flake dependencies. For example, the following sets
+the `nixpkgs` input of the top-level flake to be equal to the
+`nixpkgs` input of the `dwarffs` input of the top-level flake:
+
+```nix
+inputs.nixops.follows = "dwarffs/nixpkgs";
+```
+
+The value of the `follows` attribute is a `/`-separated sequence of
+input names denoting the path of inputs to be followed from the root
+flake.
+
+Overrides and `follows` can be combined, e.g.
+
+```nix
+inputs.nixops.inputs.nixpkgs.follows = "dwarffs/nixpkgs";
+```
+
+sets the `nixpkgs` input of `nixops` to be the same as the `nixpkgs`
+input of `dwarffs`. It is worth noting, however, that it is generally
+not useful to eliminate transitive `nixpkgs` flake inputs in this
+way. Most flakes provide their functionality through Nixpkgs overlays
+or NixOS modules, which are composed into the top-level flake's
+`nixpkgs` input; so their own `nixpkgs` input is usually irrelevant.
+
+# Lock files
+
+Inputs specified in `flake.nix` are typically "unlocked" in the sense
+that they don't specify an exact revision. To ensure reproducibility,
+Nix will automatically generate and use a *lock file* called
+`flake.lock` in the flake's directory. The lock file contains a graph
+structure isomorphic to the graph of dependencies of the root
+flake. Each node in the graph (except the root node) maps the
+(usually) unlocked input specifications in `flake.nix` to locked input
+specifications. Each node also contains some metadata, such as the
+dependencies (outgoing edges) of the node.
+
+For example, if `flake.nix` has the inputs in the example above, then
+the resulting lock file might be:
+
+```json
+{
+  "version": 7,
+  "root": "n1",
+  "nodes": {
+    "n1": {
+      "inputs": {
+        "nixpkgs": "n2",
+        "import-cargo": "n3",
+        "grcov": "n4"
+      }
+    },
+    "n2": {
+      "inputs": {},
+      "locked": {
+        "owner": "edolstra",
+        "repo": "nixpkgs",
+        "rev": "7f8d4b088e2df7fdb6b513bc2d6941f1d422a013",
+        "type": "github",
+        "lastModified": 1580555482,
+        "narHash": "sha256-OnpEWzNxF/AU4KlqBXM2s5PWvfI5/BS6xQrPvkF5tO8="
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "n3": {
+      "inputs": {},
+      "locked": {
+        "owner": "edolstra",
+        "repo": "import-cargo",
+        "rev": "8abf7b3a8cbe1c8a885391f826357a74d382a422",
+        "type": "github",
+        "lastModified": 1567183309,
+        "narHash": "sha256-wIXWOpX9rRjK5NDsL6WzuuBJl2R0kUCnlpZUrASykSc="
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "import-cargo",
+        "type": "github"
+      }
+    },
+    "n4": {
+      "inputs": {},
+      "locked": {
+        "owner": "mozilla",
+        "repo": "grcov",
+        "rev": "989a84bb29e95e392589c4e73c29189fd69a1d4e",
+        "type": "github",
+        "lastModified": 1580729070,
+        "narHash": "sha256-235uMxYlHxJ5y92EXZWAYEsEb6mm+b069GAd+BOIOxI="
+      },
+      "original": {
+        "owner": "mozilla",
+        "repo": "grcov",
+        "type": "github"
+      },
+      "flake": false
+    }
+  }
+}
+```
+
+This graph has 4 nodes: the root flake, and its 3 dependencies. The
+nodes have arbitrary labels (e.g. `n1`). The label of the root node of
+the graph is specified by the `root` attribute. Nodes contain the
+following fields:
+
+* `inputs`: The dependencies of this node, as a mapping from input
+  names (e.g. `nixpkgs`) to node labels (e.g. `n2`).
+
+* `original`: The original input specification from `flake.lock`, as a
+  set of `builtins.fetchTree` arguments.
+
+* `locked`: The locked input specification, as a set of
+  `builtins.fetchTree` arguments. Thus, in the example above, when we
+  build this flake, the input `nixpkgs` is mapped to revision
+  `7f8d4b088e2df7fdb6b513bc2d6941f1d422a013` of the `edolstra/nixpkgs`
+  repository on GitHub.
+
+  It also includes the attribute `narHash`, specifying the expected
+  contents of the tree in the Nix store (as computed by `nix
+  hash-path`), and may include input-type-specific attributes such as
+  the `lastModified` or `revCount`. The main reason for these
+  attributes is to allow flake inputs to be substituted from a binary
+  cache: `narHash` allows the store path to be computed, while the
+  other attributes are necessary because they provide information not
+  stored in the store path.
+
+* `flake`: A Boolean denoting whether this is a flake or non-flake
+  dependency. Corresponds to the `flake` attribute in the `inputs`
+  attribute in `flake.nix`.
+
+The `original` and `locked` attributes are omitted for the root
+node. This is because we cannot record the commit hash or content hash
+of the root flake, since modifying `flake.lock` will invalidate these.
+
+The graph representation of lock files allows circular dependencies
+between flakes. For example, here are two flakes that reference each
+other:
+
+```nix
+{
+  inputs.b = ... location of flake B ...;
+  # Tell the 'b' flake not to fetch 'a' again, to ensure its 'a' is
+  # *this* 'a'.
+  inputs.b.inputs.a.follows = "";
+  outputs = { self, b }: {
+    foo = 123 + b.bar;
+    xyzzy = 1000;
+  };
+}
+```
+
+and
+
+```nix
+{
+  inputs.a = ... location of flake A ...;
+  inputs.a.inputs.b.follows = "";
+  outputs = { self, a }: {
+    bar = 456 + a.xyzzy;
+  };
+}
+```
+
+Lock files transitively lock direct as well as indirect
+dependencies. That is, if a lock file exists and is up to date, Nix
+will not look at the lock files of dependencies. However, lock file
+generation itself *does* use the lock files of dependencies by
+default.
+
+)""

From 1047cb1e53358755cb12a5361cfc99118ed7e159 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Dec 2020 13:21:31 +0100
Subject: [PATCH 623/882] Command: Remove examples()

---
 doc/manual/generate-manpage.nix |  5 -----
 src/libutil/args.cc             | 30 ------------------------------
 src/libutil/args.hh             | 14 --------------
 3 files changed, 49 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 9f30c8fbc..c2c748464 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -20,11 +20,6 @@ let
            (attrNames def.commands))
          + "\n"
        else "")
-    + (if def.examples or [] != []
-       then
-         "# Examples\n\n"
-         + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
-       else "")
     + (if def ? doc
        then def.doc + "\n\n"
        else "")
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index a929ea5ac..fb5cb80fb 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -353,36 +353,6 @@ void printTable(std::ostream & out, const Table2 & table)
     }
 }
 
-void Command::printHelp(const string & programName, std::ostream & out)
-{
-    Args::printHelp(programName, out);
-
-    auto exs = examples();
-    if (!exs.empty()) {
-        out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n";
-        for (auto & ex : exs)
-            out << "\n"
-                << "  " << ex.description << "\n" // FIXME: wrap
-                << "  $ " << ex.command << "\n";
-    }
-}
-
-nlohmann::json Command::toJSON()
-{
-    auto exs = nlohmann::json::array();
-
-    for (auto & example : examples()) {
-        auto ex = nlohmann::json::object();
-        ex["description"] = example.description;
-        ex["command"] = chomp(stripIndentation(example.command));
-        exs.push_back(std::move(ex));
-    }
-
-    auto res = Args::toJSON();
-    res["examples"] = std::move(exs);
-    return res;
-}
-
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 68bbbb4f7..6ed541a32 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -228,25 +228,11 @@ struct Command : virtual Args
     virtual void prepare() { };
     virtual void run() = 0;
 
-    struct Example
-    {
-        std::string description;
-        std::string command;
-    };
-
-    typedef std::list<Example> Examples;
-
-    virtual Examples examples() { return Examples(); }
-
     typedef int Category;
 
     static constexpr Category catDefault = 0;
 
     virtual Category category() { return catDefault; }
-
-    void printHelp(const string & programName, std::ostream & out) override;
-
-    nlohmann::json toJSON() override;
 };
 
 typedef std::map<std::string, std::function<ref<Command>()>> Commands;

From 26e502ceb567e48f3eb28fb1848430ab69cb7606 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Dec 2020 16:41:07 +0100
Subject: [PATCH 624/882] Add TODO

---
 src/nix/bundle.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/bundle.md b/src/nix/bundle.md
index c183a170d..5e2298376 100644
--- a/src/nix/bundle.md
+++ b/src/nix/bundle.md
@@ -29,4 +29,8 @@ for more details.
 >
 > This command only works on Linux.
 
+# Bundler definitions
+
+TODO
+
 )""

From 5178211e963fa111f84c4881b22cc506d5254fde Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Dec 2020 18:33:42 +0100
Subject: [PATCH 625/882] Add 'nix' manpage

---
 src/nix/main.cc |   7 +++
 src/nix/nix.md  | 119 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+)
 create mode 100644 src/nix/nix.md

diff --git a/src/nix/main.cc b/src/nix/main.cc
index afe7cb8d7..b2406fafe 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -184,6 +184,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     {
         return "a tool for reproducible and declarative configuration management";
     }
+
+    std::string doc() override
+    {
+        return
+          #include "nix.md"
+          ;
+    }
 };
 
 static void showHelp(std::vector<std::string> subcommand)
diff --git a/src/nix/nix.md b/src/nix/nix.md
new file mode 100644
index 000000000..d10de7c01
--- /dev/null
+++ b/src/nix/nix.md
@@ -0,0 +1,119 @@
+R""(
+
+# Examples
+
+* Create a new flake:
+
+  ```console
+  # nix flake new hello
+  # cd hello
+  ```
+
+* Build the flake in the current directory:
+
+  ```console
+  # nix build
+  # ./result/bin/hello
+  Hello, world!
+  ```
+
+* Run the flake in the current directory:
+
+  ```console
+  # nix run
+  Hello, world!
+  ```
+
+* Start a development shell for hacking on this flake:
+
+  ```console
+  # nix develop
+  # unpackPhase
+  # cd hello-*
+  # configurePhase
+  # buildPhase
+  # ./hello
+  Hello, world!
+  # installPhase
+  # ../outputs/out/bin/hello
+  Hello, world!
+  ```
+
+# Description
+
+Nix is a tool for building software, configurations and other
+artifacts in a reproducible and declarative way. For more information,
+see the [Nix homepage](https://nixos.org/) or the [Nix
+manual](https://nixos.org/manual/nix/stable/).
+
+# Installables
+
+Many `nix` subcommands operate on one or more *installables*. These are
+command line arguments that represent something that can be built in
+the Nix store. Here are the recognised types of installables:
+
+* **Flake output attributes**: `nixpkgs#hello`
+
+  These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a
+  flake reference and *attrpath* is an optional attribute path. For
+  more information on flakes, see [the `nix flake` manual
+  page](./nix3-flake.md). Flake references are most commonly a flake
+  identifier in the flake registry (e.g. `nixpkgs`) or a path
+  (e.g. `/path/to/my-flake` or `.`).
+
+  If *attrpath* is omitted, Nix tries some default values; for most
+  subcommands, the default is `defaultPackage.`*system*
+  (e.g. `defaultPackage.x86_64-linux`), but some subcommands have
+  other defaults. If *attrpath* *is* specified, *attrpath* is
+  interpreted as relative to one or more prefixes; for most
+  subcommands, these are `packages.`*system*,
+  `legacyPackages.*system*` and the empty prefix. Thus, on
+  `x86_64-linux` `nix build nixpkgs#hello` will try to build the
+  attributes `packages.x86_64-linux.hello`,
+  `legacyPackages.x86_64-linux.hello` and `hello`.
+
+* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10`
+
+  These are paths inside the Nix store, or symlinks that resolve to a
+  path in the Nix store.
+
+* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv`
+
+  Store derivations are store paths with extension `.drv` and are a
+  low-level representation of a build-time dependency graph used
+  internally by Nix. By default, if you pass a store derivation to a
+  `nix` subcommand, it will operate on the *output paths* of the
+  derivation. For example, `nix path-info` prints information about
+  the output paths:
+
+  ```console
+  # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv
+  [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}]
+  ```
+
+  If you want to operate on the store derivation itself, pass the
+  `--derivation` flag.
+
+* **Nix attributes**: `--file /path/to/nixpkgs hello`
+
+  When the `-f` / `--file` *path* option is given, installables are
+  interpreted as attribute paths referencing a value returned by
+  evaluating the Nix file *path*.
+
+* **Nix expressions**: `--expr '(import <nixpkgs> {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`.
+
+  When the `--expr` option is given, all installables are interpreted
+  as Nix expressions. You may need to specify `--impure` if the
+  expression references impure inputs (such as `<nixpkgs>`).
+
+For most commands, if no installable is specified, the default is `.`,
+i.e. Nix will operate on the default flake output attribute of the
+flake in the current directory.
+
+# Nix stores
+
+Most `nix` subcommands operate on a *Nix store*.
+
+TODO: list store types, options
+
+)""

From 29bd63e9907cabc5643aaa3f570b9ff5b2d88268 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 19:55:21 +0000
Subject: [PATCH 626/882] Test nix-instantiate with binary cache store

Trying to make sure it work with obscurers stores.
---
 tests/binary-cache.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 92ed36225..8f1c6f14d 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -1,15 +1,20 @@
 source common.sh
 
+# We can produce drvs directly into the binary cache
 clearStore
-clearCache
+clearCacheCache
+nix-instantiate --store "file://$cacheDir" dependencies.nix
 
 # Create the binary cache.
+clearStore
+clearCache
 outPath=$(nix-build dependencies.nix --no-out-link)
 
 nix copy --to file://$cacheDir $outPath
 
 
-basicTests() {
+basicDownloadTests() {
+    # No uploading tests bcause upload with force HTTP doesn't work.
 
     # By default, a binary cache doesn't support "nix-env -qas", but does
     # support installation.
@@ -44,12 +49,12 @@ basicTests() {
 
 
 # Test LocalBinaryCacheStore.
-basicTests
+basicDownloadTests
 
 
 # Test HttpBinaryCacheStore.
 export _NIX_FORCE_HTTP=1
-basicTests
+basicDownloadTests
 
 
 # Test whether Nix notices if the NAR doesn't match the hash in the NAR info.

From 57062179ce36e35715284d2ef570f8cb0b90198d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 16:05:09 +0000
Subject: [PATCH 627/882] Move some PKI stuff from LocalStore to Store

---
 src/libstore/local-store.cc |  9 ---------
 src/libstore/local-store.hh | 12 ------------
 src/libstore/misc.cc        |  9 +++++++++
 src/libstore/store-api.hh   | 13 +++++++++++++
 4 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c52d4b62a..1eb2dec75 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1092,15 +1092,6 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 }
 
 
-const PublicKeys & LocalStore::getPublicKeys()
-{
-    auto state(_state.lock());
-    if (!state->publicKeys)
-        state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys());
-    return *state->publicKeys;
-}
-
-
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index ae9497b2e..d97645058 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -35,10 +35,6 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 {
     using LocalFSStoreConfig::LocalFSStoreConfig;
 
-    Setting<bool> requireSigs{(StoreConfig*) this,
-        settings.requireSigs,
-        "require-sigs", "whether store paths should have a trusted signature on import"};
-
     const std::string name() override { return "Local Store"; }
 };
 
@@ -75,8 +71,6 @@ private:
            minFree but not much below availAfterGC, then there is no
            point in starting a new GC. */
         uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
-
-        std::unique_ptr<PublicKeys> publicKeys;
     };
 
     Sync<State> _state;
@@ -94,12 +88,6 @@ public:
     const Path tempRootsDir;
     const Path fnTempRoots;
 
-private:
-
-    const PublicKeys & getPublicKeys();
-
-public:
-
     // Hack for build-remote.cc.
     PathSet locksHeld;
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index ad4dccef9..0d4190a56 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -282,4 +282,13 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
 }
 
 
+const PublicKeys & Store::getPublicKeys()
+{
+    auto cryptoState(_cryptoState.lock());
+    if (!cryptoState->publicKeys)
+        cryptoState->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys());
+    return *cryptoState->publicKeys;
+}
+
+
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9bcff08eb..e3de6db17 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -189,6 +189,10 @@ struct StoreConfig : public Config
 
     const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
 
+    Setting<bool> requireSigs{this,
+        settings.requireSigs,
+        "require-sigs", "whether store paths should have a trusted signature on import"};
+
     Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
@@ -710,11 +714,20 @@ public:
         return toRealPath(printStorePath(storePath));
     }
 
+    const PublicKeys & getPublicKeys();
+
     virtual void createUser(const std::string & userName, uid_t userId)
     { }
 
 protected:
 
+    struct CryptoState
+    {
+        std::unique_ptr<PublicKeys> publicKeys;
+    };
+
+    Sync<CryptoState> _cryptoState;
+
     Stats stats;
 
     /* Unsupported methods. */

From 12f7a1f65becfe3b036d0f840ee4a05f2f1f857c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 17:07:28 +0000
Subject: [PATCH 628/882] build-remote no longer requires local store be local

---
 src/build-remote/build-remote.cc | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8348d8c91..350bd6cef 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -71,11 +71,15 @@ static int main_build_remote(int argc, char * * argv)
 
         initPlugins();
 
-        auto store = openStore().cast<LocalStore>();
+        auto store = openStore();
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
-        currentLoad = store->stateDir + "/current-load";
+        currentLoad = "/current-load";
+        if (auto localStore = store.dynamic_pointer_cast<LocalFSStore>())
+            currentLoad = std::string { localStore->stateDir } + currentLoad;
+        else
+            currentLoad = settings.nixStateDir + currentLoad;
 
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;
@@ -288,8 +292,9 @@ connected:
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            for (auto & i : missing)
-                store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
+            if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
+                for (auto & i : missing)
+                    localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 

From 450c3500f1e3fb619636c0a29d65300020f99d7d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 17:36:52 +0000
Subject: [PATCH 629/882] Crudely make worker only provide a Store, not
 LocalStore

We downcast in a few places, this will be refactored to be better later.
---
 src/libstore/build/derivation-goal.cc | 66 ++++++++++++++++++---------
 src/libstore/build/worker.cc          |  6 ++-
 src/libstore/build/worker.hh          |  9 ++--
 3 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 47d11dc53..de32f60db 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -848,14 +848,16 @@ void DerivationGoal::buildDone()
                So instead, check if the disk is (nearly) full now.  If
                so, we don't mark this build as a permanent failure. */
 #if HAVE_STATVFS
-            uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-            struct statvfs st;
-            if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
-            if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                (uint64_t) st.f_bavail * st.f_bsize < required)
-                diskFull = true;
+            if (auto localStore = dynamic_cast<LocalStore *>(&worker.store)) {
+                uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
+                struct statvfs st;
+                if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 &&
+                    (uint64_t) st.f_bavail * st.f_bsize < required)
+                    diskFull = true;
+                if (statvfs(tmpDir.c_str(), &st) == 0 &&
+                    (uint64_t) st.f_bavail * st.f_bsize < required)
+                    diskFull = true;
+            }
 #endif
 
             deleteTmpDir(false);
@@ -1215,12 +1217,15 @@ void DerivationGoal::startBuilder()
             useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
     }
 
-    if (worker.store.storeDir != worker.store.realStoreDir) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
+    if (auto localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
+        auto & localStore = *localStoreP;
+        if (localStore.storeDir != localStore.realStoreDir) {
+            #if __linux__
+                useChroot = true;
+            #else
+                throw Error("building using a diverted store is not supported on this platform");
+            #endif
+        }
     }
 
     /* Create a temporary directory where the build will take
@@ -2182,7 +2187,8 @@ void DerivationGoal::startDaemon()
     Store::Params params;
     params["path-info-cache-size"] = "0";
     params["store"] = worker.store.storeDir;
-    params["root"] = worker.store.rootDir;
+    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
+        params["root"] = localStore->rootDir;
     params["state"] = "/no-such-path";
     params["log"] = "/no-such-path";
     auto store = make_ref<RestrictedStore>(params,
@@ -3246,7 +3252,13 @@ void DerivationGoal::registerOutputs()
             }
         }
 
+        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
+        if (!localStoreP)
+            Unsupported("Can only register outputs with local store");
+        auto & localStore = *localStoreP;
+
         if (buildMode == bmCheck) {
+
             if (!worker.store.isValidPath(newInfo.path)) continue;
             ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
             if (newInfo.narHash != oldInfo.narHash) {
@@ -3271,8 +3283,8 @@ void DerivationGoal::registerOutputs()
             /* Since we verified the build, it's now ultimately trusted. */
             if (!oldInfo.ultimate) {
                 oldInfo.ultimate = true;
-                worker.store.signPathInfo(oldInfo);
-                worker.store.registerValidPaths({{oldInfo.path, oldInfo}});
+                localStore.signPathInfo(oldInfo);
+                localStore.registerValidPaths({{oldInfo.path, oldInfo}});
             }
 
             continue;
@@ -3288,13 +3300,13 @@ void DerivationGoal::registerOutputs()
         }
 
         if (curRound == nrRounds) {
-            worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
+            localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
             worker.markContentsGood(newInfo.path);
         }
 
         newInfo.deriver = drvPath;
         newInfo.ultimate = true;
-        worker.store.signPathInfo(newInfo);
+        localStore.signPathInfo(newInfo);
 
         finish(newInfo.path);
 
@@ -3302,7 +3314,7 @@ void DerivationGoal::registerOutputs()
            isn't statically known so that we can safely unlock the path before
            the next iteration */
         if (newInfo.ca)
-            worker.store.registerValidPaths({{newInfo.path, newInfo}});
+            localStore.registerValidPaths({{newInfo.path, newInfo}});
 
         infos.emplace(outputName, std::move(newInfo));
     }
@@ -3375,11 +3387,16 @@ void DerivationGoal::registerOutputs()
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
     {
+        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
+        if (!localStoreP)
+            Unsupported("Can only register outputs with local store");
+        auto & localStore = *localStoreP;
+
         ValidPathInfos infos2;
         for (auto & [outputName, newInfo] : infos) {
             infos2.insert_or_assign(newInfo.path, newInfo);
         }
-        worker.store.registerValidPaths(infos2);
+        localStore.registerValidPaths(infos2);
     }
 
     /* In case of a fixed-output derivation hash mismatch, throw an
@@ -3577,7 +3594,12 @@ Path DerivationGoal::openLogFile()
     auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
 
     /* Create a log file. */
-    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
+    Path logDir;
+    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
+        logDir = localStore->logDir;
+    else
+        logDir = settings.nixLogDir;
+    Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
     createDirs(dir);
 
     Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 6c96a93bd..a9575fb0f 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -8,7 +8,7 @@
 
 namespace nix {
 
-Worker::Worker(LocalStore & store)
+Worker::Worker(Store & store)
     : act(*logger, actRealise)
     , actDerivations(*logger, actBuilds)
     , actSubstitutions(*logger, actCopyPaths)
@@ -229,7 +229,9 @@ void Worker::run(const Goals & _topGoals)
 
         checkInterrupt();
 
-        store.autoGC(false);
+        // TODO GC interface?
+        if (auto localStore = dynamic_cast<LocalStore *>(&store))
+            localStore->autoGC(false);
 
         /* Call every wake goal (in the ordering established by
            CompareGoalPtrs). */
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index bf8cc4586..82e711191 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -2,9 +2,12 @@
 
 #include "types.hh"
 #include "lock.hh"
-#include "local-store.hh"
+#include "store-api.hh"
 #include "goal.hh"
 
+#include <future>
+#include <thread>
+
 namespace nix {
 
 /* Forward definition. */
@@ -102,7 +105,7 @@ public:
     /* Set if at least one derivation is not deterministic in check mode. */
     bool checkMismatch;
 
-    LocalStore & store;
+    Store & store;
 
     std::unique_ptr<HookInstance> hook;
 
@@ -124,7 +127,7 @@ public:
        it answers with "decline-permanently", we don't try again. */
     bool tryBuildHook = true;
 
-    Worker(LocalStore & store);
+    Worker(Store & store);
     ~Worker();
 
     /* Make a goal (with caching). */

From 85f2e9e8fa4f7452a05cfffc901d118a7c861d0a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 17:54:57 +0000
Subject: [PATCH 630/882] Expose schedule entrypoints to all stores

Remote stores still override so the other end schedules.
---
 src/libstore/binary-cache-store.hh            |  7 ------
 .../{local-store-build.cc => entry-points.cc} |  6 ++---
 src/libstore/dummy-store.cc                   |  7 ------
 src/libstore/local-store.hh                   |  9 --------
 src/libstore/store-api.cc                     | 23 -------------------
 src/libstore/store-api.hh                     |  6 ++---
 6 files changed, 6 insertions(+), 52 deletions(-)
 rename src/libstore/build/{local-store-build.cc => entry-points.cc} (91%)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 443a53cac..c2163166c 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -108,13 +108,6 @@ public:
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode) override
-    { unsupported("buildDerivation"); }
-
-    void ensurePath(const StorePath & path) override
-    { unsupported("ensurePath"); }
-
     ref<FSAccessor> getFSAccessor() override;
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/entry-points.cc
similarity index 91%
rename from src/libstore/build/local-store-build.cc
rename to src/libstore/build/entry-points.cc
index c91cda2fd..9f97d40ba 100644
--- a/src/libstore/build/local-store-build.cc
+++ b/src/libstore/build/entry-points.cc
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
 {
     Worker worker(*this);
 
@@ -43,7 +43,7 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
     }
 }
 
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
     Worker worker(*this);
@@ -63,7 +63,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe
 }
 
 
-void LocalStore::ensurePath(const StorePath & path)
+void Store::ensurePath(const StorePath & path)
 {
     /* If the path is already valid, we're done. */
     if (isValidPath(path)) return;
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 3c7caf8f2..8f26af685 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -55,13 +55,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
     void narFromPath(const StorePath & path, Sink & sink) override
     { unsupported("narFromPath"); }
 
-    void ensurePath(const StorePath & path) override
-    { unsupported("ensurePath"); }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode) override
-    { unsupported("buildDerivation"); }
-
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override
     { unsupported("queryRealisation"); }
 };
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index d97645058..aa5de31f0 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -133,15 +133,6 @@ public:
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
 
-    void buildPaths(
-        const std::vector<StorePathWithOutputs> & paths,
-        BuildMode buildMode) override;
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode) override;
-
-    void ensurePath(const StorePath & path) override;
-
     void addTempRoot(const StorePath & path) override;
 
     void addIndirectRoot(const Path & path) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7aca22bde..f12a564a1 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -747,29 +747,6 @@ const Store::Stats & Store::getStats()
 }
 
 
-void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode)
-{
-    StorePathSet paths2;
-
-    for (auto & path : paths) {
-        if (path.path.isDerivation()) {
-            auto outPaths = queryPartialDerivationOutputMap(path.path);
-            for (auto & outputName : path.outputs) {
-                auto currentOutputPathIter = outPaths.find(outputName);
-                if (currentOutputPathIter == outPaths.end() ||
-                    !currentOutputPathIter->second ||
-                    !isValidPath(*currentOutputPathIter->second))
-                    unsupported("buildPaths");
-            }
-        } else
-            paths2.insert(path.path);
-    }
-
-    if (queryValidPaths(paths2).size() != paths2.size())
-        unsupported("buildPaths");
-}
-
-
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e3de6db17..4db980fe9 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -523,17 +523,17 @@ public:
            explicitly choosing to allow it).
     */
     virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) = 0;
+        BuildMode buildMode = bmNormal);
 
     /* Ensure that a path is valid.  If it is not currently valid, it
        may be made valid by running a substitute (if defined for the
        path). */
-    virtual void ensurePath(const StorePath & path) = 0;
+    virtual void ensurePath(const StorePath & path);
 
     /* Add a store path as a temporary root of the garbage collector.
        The root disappears as soon as we exit. */
     virtual void addTempRoot(const StorePath & path)
-    { unsupported("addTempRoot"); }
+    { warn("not creating temp root, store doesn't support GC"); }
 
     /* Add an indirect root, which is merely a symlink to `path' from
        /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed

From fed123724679de89d3f56a4c01b5c4c96f93e584 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Dec 2020 19:55:21 +0000
Subject: [PATCH 631/882] Test nix-build with non-local-store --store

Just a few small things needed fixing!
---
 src/libstore/build/derivation-goal.cc | 20 +++++++++++++++++---
 tests/binary-cache-build-remote.sh    | 13 +++++++++++++
 tests/local.mk                        |  4 +++-
 3 files changed, 33 insertions(+), 4 deletions(-)
 create mode 100644 tests/binary-cache-build-remote.sh

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index de32f60db..17f39a86e 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -592,9 +592,17 @@ void DerivationGoal::tryToBuild()
     PathSet lockFiles;
     /* FIXME: Should lock something like the drv itself so we don't build same
        CA drv concurrently */
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+    if (dynamic_cast<LocalStore *>(&worker.store))
+        /* If we aren't a local store, we might need to use the local store as
+           a build remote, but that would cause a deadlock. */
+        /* FIXME: Make it so we can use ourselves as a build remote even if we
+           are the local store (separate locking for building vs scheduling? */
+        /* FIXME: find some way to lock for scheduling for the other stores so
+           a forking daemon with --store still won't farm out redundant builds.
+           */
+        for (auto & i : drv->outputsAndOptPaths(worker.store))
+            if (i.second.second)
+                lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         if (!actLock)
@@ -680,6 +688,12 @@ void DerivationGoal::tryLocalBuild() {
     /* Make sure that we are allowed to start a build.  If this
        derivation prefers to be done locally, do it even if
        maxBuildJobs is 0. */
+    if (!dynamic_cast<LocalStore *>(&worker.store)) {
+        throw Error(
+            "unable to build with a primary store that isn't a local store; "
+            "either pass a different '--store' or enable remote builds."
+            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+    }
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
         worker.waitForBuildSlot(shared_from_this());
diff --git a/tests/binary-cache-build-remote.sh b/tests/binary-cache-build-remote.sh
new file mode 100644
index 000000000..ed51164a4
--- /dev/null
+++ b/tests/binary-cache-build-remote.sh
@@ -0,0 +1,13 @@
+source common.sh
+
+clearStore
+clearCacheCache
+
+# Fails without remote builders
+(! nix-build --store "file://$cacheDir" dependencies.nix)
+
+# Succeeds with default store as build remote.
+nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix
+
+# Succeeds without any build capability because no-op
+nix-build --store "file://$cacheDir" -j0 dependencies.nix
diff --git a/tests/local.mk b/tests/local.mk
index ce94ec80e..aa8b4f9bf 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -9,7 +9,9 @@ nix_tests = \
   local-store.sh remote-store.sh export.sh export-graph.sh \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
-  binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
+  binary-cache.sh \
+  binary-cache-build-remote.sh \
+  nix-profile.sh repair.sh dump-db.sh case-hack.sh \
   check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \

From d4870462f8f539adeaa6dca476aff6f1f31e1981 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 8 Dec 2020 14:16:06 -0600
Subject: [PATCH 632/882] Cast variants fully for libc++10

libc++10 seems to be stricter on what it allows in variant conversion.
I'm not sure what the rules are here, but this is the minimal change
needed to get through the compilation errors.
---
 src/libexpr/eval-cache.cc        | 2 +-
 src/libexpr/primops/fetchTree.cc | 2 +-
 src/libfetchers/attrs.cc         | 2 +-
 src/libfetchers/github.cc        | 4 ++--
 src/libfetchers/mercurial.cc     | 2 +-
 src/libfetchers/tarball.cc       | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 7b025be23..539ba71f3 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -394,7 +394,7 @@ Value & AttrCursor::forceValue()
             cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
                            string_t{v.string.s, {}}};
         else if (v.type == tPath)
-            cachedValue = {root->db->setString(getKey(), v.path), v.path};
+            cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
         else if (v.type == tBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
         else if (v.type == tAttrs)
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index d094edf92..1360ade39 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -104,7 +104,7 @@ static void fetchTree(
             else if (attr.value->type == tBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
             else if (attr.value->type == tInt)
-                attrs.emplace(attr.name, attr.value->integer);
+                attrs.emplace(attr.name, uint64_t(attr.value->integer));
             else
                 throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                     attr.name, showType(*attr.value));
diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index 720b19fcd..17fc4041f 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -11,7 +11,7 @@ Attrs jsonToAttrs(const nlohmann::json & json)
 
     for (auto & i : json.items()) {
         if (i.value().is_number())
-            attrs.emplace(i.key(), i.value().get<int64_t>());
+            attrs.emplace(i.key(), i.value().get<uint64_t>());
         else if (i.value().is_string())
             attrs.emplace(i.key(), i.value().get<std::string>());
         else if (i.value().is_boolean())
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 92ff224f7..db1ced5d6 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -195,14 +195,14 @@ struct GitArchiveInputScheme : InputScheme
 
         auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", lastModified);
+        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", lastModified}
+                {"lastModified", uint64_t(lastModified)}
             },
             tree.storePath,
             true);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 07a51059d..0eb401e10 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},
-            {"revCount", (int64_t) revCount},
+            {"revCount", (uint64_t) revCount},
         });
 
         if (!_input.getRev())
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 8c0f20475..56c014a8c 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -152,7 +152,7 @@ std::pair<Tree, time_t> downloadTarball(
     }
 
     Attrs infoAttrs({
-        {"lastModified", lastModified},
+        {"lastModified", uint64_t(lastModified)},
         {"etag", res.etag},
     });
 

From 9d3aad7b92762973081738a8bc6a562fda45c341 Mon Sep 17 00:00:00 2001
From: Sevan Janiyan <venture37@geeklan.co.uk>
Date: Fri, 25 Dec 2020 01:43:22 +0000
Subject: [PATCH 633/882] Update URL where bzip2 can be obtained

---
 doc/manual/src/installation/prerequisites-source.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index 69b7c5a5e..6825af707 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -30,7 +30,7 @@
     have bzip2 installed, including development headers and libraries.
     If your distribution does not provide these, you can obtain bzip2
     from
-    <https://web.archive.org/web/20180624184756/http://www.bzip.org/>.
+    <https://sourceware.org/bzip2/>.
 
   - `liblzma`, which is provided by XZ Utils. If your distribution does
     not provide this, you can get it from <https://tukaani.org/xz/>.

From f1e9bda9d1ece9be8c78f5c5345c3adb299bc4aa Mon Sep 17 00:00:00 2001
From: Sevan Janiyan <venture37@geeklan.co.uk>
Date: Fri, 25 Dec 2020 01:48:21 +0000
Subject: [PATCH 634/882] Update URL where bzip2 can be obtained

---
 configure.ac | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index daf378997..c1bfc9b53 100644
--- a/configure.ac
+++ b/configure.ac
@@ -174,9 +174,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]
 
 # Look for libbz2, a required dependency.
 AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 # Checks for libarchive
 PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed

From 6262a703636000e525d5c1b877ac28d604a493f0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Dec 2020 17:21:19 +0100
Subject: [PATCH 635/882] scanForReferences: Remove misleading comment

References have always been determined only by the hash part, not the
name or the store prefix.

Fixes #4396.
---
 src/libstore/references.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index eb117b5ba..39c4970c6 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee,
     TeeSink sink { refsSink, toTee };
     std::map<string, Path> backMap;
 
-    /* For efficiency (and a higher hit rate), just search for the
-       hash part of the file name.  (This assumes that all references
-       have the form `HASH-bla'). */
     for (auto & i : refs) {
         auto baseName = std::string(baseNameOf(i));
         string::size_type pos = baseName.find('-');

From 093de16223b8b93d803e4cd1cc1d3945cb3dfeb1 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 28 Dec 2020 09:30:14 -0800
Subject: [PATCH 636/882] README: fix link to hacking guide

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 11fe5f932..4686010ef 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download
 
 ## Building And Developing
 
-See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/hacking.html) in our manual for instruction on how to
+See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to
 build nix from source with nix-build or how to get a development environment.
 
 ## Additional Resources

From 64904b9d5d32c4201aaf462ae82b736f33785793 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Mon, 28 Dec 2020 19:40:04 -0600
Subject: [PATCH 637/882] Fixup

---
 src/libexpr/eval-cache.cc        | 4 ++--
 src/libexpr/primops/fetchTree.cc | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 0f84944cd..98d91c905 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -393,9 +393,9 @@ Value & AttrCursor::forceValue()
         if (v.type() == nString)
             cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
                            string_t{v.string.s, {}}};
-        else if (v.type == nPath)
+        else if (v.type() == nPath)
             cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
-        else if (v.type == nBool)
+        else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index e64e3fbb8..ab80be2d3 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -103,7 +103,7 @@ static void fetchTree(
                 addURI(state, attrs, attr.name, attr.value->string.s);
             else if (attr.value->type() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
-            else if (attr.value->type == nInt)
+            else if (attr.value->type() == nInt)
                 attrs.emplace(attr.name, uint64_t(attr.value->integer));
             else
                 throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",

From d27eb0ef573b4739967119448779da4a8b2a2cbf Mon Sep 17 00:00:00 2001
From: David McFarland <corngood@gmail.com>
Date: Wed, 30 Dec 2020 16:20:03 -0400
Subject: [PATCH 638/882] Fix insufficent attribute capacity in user profile

---
 src/nix-env/user-env.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 87387e794..168ac492b 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -53,10 +53,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
            output paths, and optionally the derivation path, as well
            as the meta attributes. */
         Path drvPath = keepDerivations ? i.queryDrvPath() : "";
+        DrvInfo::Outputs outputs = i.queryOutputs(true);
+        StringSet metaNames = i.queryMetaNames();
 
         Value & v(*state.allocValue());
         manifest.listElems()[n++] = &v;
-        state.mkAttrs(v, 16);
+        state.mkAttrs(v, 7 + outputs.size());
 
         mkString(*state.allocAttr(v, state.sType), "derivation");
         mkString(*state.allocAttr(v, state.sName), i.queryName());
@@ -68,7 +70,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
             mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath());
 
         // Copy each output meant for installation.
-        DrvInfo::Outputs outputs = i.queryOutputs(true);
         Value & vOutputs = *state.allocAttr(v, state.sOutputs);
         state.mkList(vOutputs, outputs.size());
         unsigned int m = 0;
@@ -88,8 +89,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
         // Copy the meta attributes.
         Value & vMeta = *state.allocAttr(v, state.sMeta);
-        state.mkAttrs(vMeta, 16);
-        StringSet metaNames = i.queryMetaNames();
+        state.mkAttrs(vMeta, metaNames.size());
         for (auto & j : metaNames) {
             Value * v = i.queryMeta(j);
             if (!v) continue;

From e069ddf3258b7eab2074639cade03ba03b0a03a4 Mon Sep 17 00:00:00 2001
From: Sam Lidder <sam.lidder@gmail.com>
Date: Thu, 31 Dec 2020 20:17:37 -0500
Subject: [PATCH 639/882] Fix `configure` error in introduction doc

---
 doc/manual/src/introduction.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md
index f01fe7b38..d68445c95 100644
--- a/doc/manual/src/introduction.md
+++ b/doc/manual/src/introduction.md
@@ -165,10 +165,10 @@ You’re then dropped into a shell where you can edit, build and test
 the package:
 
 ```console
-[nix-shell]$ tar xf $src
+[nix-shell]$ unpackPhase
 [nix-shell]$ cd pan-*
-[nix-shell]$ ./configure
-[nix-shell]$ make
+[nix-shell]$ configurePhase
+[nix-shell]$ buildPhase
 [nix-shell]$ ./pan/gui/pan
 ```
 

From 988dd0a65f562741708f6a7a7a44e333d6a5b205 Mon Sep 17 00:00:00 2001
From: Danila Fedorin <danila.fedorin@gmail.com>
Date: Tue, 5 Jan 2021 02:06:25 +0000
Subject: [PATCH 640/882] Fix conversion from JSON to fetch attributes

It appears as through the fetch attribute, which
is simply a variant with 3 elements, implicitly
converts boolean arguments to integers. One must
use Explicit<bool> to correctly populate it with
a boolean. This was missing from the implementation,
and resulted in clearly boolean JSON fields being
treated as numbers.
---
 src/libfetchers/attrs.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index 17fc4041f..a565d19d4 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -15,7 +15,7 @@ Attrs jsonToAttrs(const nlohmann::json & json)
         else if (i.value().is_string())
             attrs.emplace(i.key(), i.value().get<std::string>());
         else if (i.value().is_boolean())
-            attrs.emplace(i.key(), i.value().get<bool>());
+            attrs.emplace(i.key(), Explicit<bool> { i.value().get<bool>() });
         else
             throw Error("unsupported input attribute type in lock file");
     }

From 8af4f886e212346afdd1d40789f96f1321da96c5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jan 2021 11:47:29 +0100
Subject: [PATCH 641/882] Fix deadlock in LocalStore::addSignatures()

Fixes #4367.
---
 src/libstore/local-store.cc | 95 +++++++++++++++++++------------------
 src/libstore/local-store.hh |  2 +
 2 files changed, 52 insertions(+), 45 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c52d4b62a..702e7b136 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -736,57 +736,62 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
+        callback(retrySQLite<std::shared_ptr<const ValidPathInfo>>([&]() {
             auto state(_state.lock());
-
-            /* Get the path info. */
-            auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path)));
-
-            if (!useQueryPathInfo.next())
-                return std::shared_ptr<ValidPathInfo>();
-
-            auto id = useQueryPathInfo.getInt(0);
-
-            auto narHash = Hash::dummy;
-            try {
-                narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
-            } catch (BadHash & e) {
-                throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
-            }
-
-            auto info = std::make_shared<ValidPathInfo>(path, narHash);
-
-            info->id = id;
-
-            info->registrationTime = useQueryPathInfo.getInt(2);
-
-            auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3);
-            if (s) info->deriver = parseStorePath(s);
-
-            /* Note that narSize = NULL yields 0. */
-            info->narSize = useQueryPathInfo.getInt(4);
-
-            info->ultimate = useQueryPathInfo.getInt(5) == 1;
-
-            s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6);
-            if (s) info->sigs = tokenizeString<StringSet>(s, " ");
-
-            s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7);
-            if (s) info->ca = parseContentAddressOpt(s);
-
-            /* Get the references. */
-            auto useQueryReferences(state->stmts->QueryReferences.use()(info->id));
-
-            while (useQueryReferences.next())
-                info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
-
-            return info;
+            return queryPathInfoInternal(*state, path);
         }));
 
     } catch (...) { callback.rethrow(); }
 }
 
 
+std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & state, const StorePath & path)
+{
+    /* Get the path info. */
+    auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path)));
+
+    if (!useQueryPathInfo.next())
+        return std::shared_ptr<ValidPathInfo>();
+
+    auto id = useQueryPathInfo.getInt(0);
+
+    auto narHash = Hash::dummy;
+    try {
+        narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
+    } catch (BadHash & e) {
+        throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
+    }
+
+    auto info = std::make_shared<ValidPathInfo>(path, narHash);
+
+    info->id = id;
+
+    info->registrationTime = useQueryPathInfo.getInt(2);
+
+    auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3);
+    if (s) info->deriver = parseStorePath(s);
+
+    /* Note that narSize = NULL yields 0. */
+    info->narSize = useQueryPathInfo.getInt(4);
+
+    info->ultimate = useQueryPathInfo.getInt(5) == 1;
+
+    s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6);
+    if (s) info->sigs = tokenizeString<StringSet>(s, " ");
+
+    s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
+    if (s) info->ca = parseContentAddressOpt(s);
+
+    /* Get the references. */
+    auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
+
+    while (useQueryReferences.next())
+        info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
+
+    return info;
+}
+
+
 /* Update path info in the database. */
 void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
@@ -1608,7 +1613,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
 
         SQLiteTxn txn(state->db);
 
-        auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath)));
+        auto info = std::const_pointer_cast<ValidPathInfo>(queryPathInfoInternal(*state, storePath));
 
         info->sigs.insert(sigs.begin(), sigs.end());
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index ae9497b2e..6d29c5960 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -235,6 +235,8 @@ private:
     void verifyPath(const Path & path, const StringSet & store,
         PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
 
+    std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path);
+
     void updatePathInfo(State & state, const ValidPathInfo & info);
 
     void upgradeStore6();

From 146af4ee9bb03968a7322a1ac70dc60c8d5a35e2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Jan 2021 16:43:09 +0100
Subject: [PATCH 642/882] Move sodium_init() call

---
 src/libmain/shared.cc      | 9 +++++++++
 src/nix-store/nix-store.cc | 3 ---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 2247aeca4..e9f067e35 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -18,6 +18,10 @@
 
 #include <openssl/crypto.h>
 
+#if HAVE_SODIUM
+#include <sodium.h>
+#endif
+
 
 namespace nix {
 
@@ -126,6 +130,11 @@ void initNix()
     CRYPTO_set_locking_callback(opensslLockCallback);
 #endif
 
+#if HAVE_SODIUM
+    if (sodium_init() == -1)
+        throw Error("could not initialise libsodium");
+#endif
+
     loadConfFile();
 
     startSignalHandlerThread();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 54394e921..e1ccece99 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -981,9 +981,6 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
     string publicKeyFile = *i++;
 
 #if HAVE_SODIUM
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
     unsigned char pk[crypto_sign_PUBLICKEYBYTES];
     unsigned char sk[crypto_sign_SECRETKEYBYTES];
     if (crypto_sign_keypair(pk, sk) != 0)

From 555152ffe8494190ca42dd481991c9b54759f686 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Jan 2021 17:04:46 +0100
Subject: [PATCH 643/882] crypto.cc: API cleanup and add generate() /
 to_string() methods

---
 src/libstore/crypto.cc     | 33 ++++++++++++++++++++++++++-------
 src/libstore/crypto.hh     | 24 ++++++++++++++++--------
 src/nix-store/nix-store.cc | 17 +++--------------
 3 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 9ec8abd22..135ced277 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -8,15 +8,15 @@
 
 namespace nix {
 
-static std::pair<std::string, std::string> split(const string & s)
+static std::pair<std::string_view, std::string_view> split(std::string_view s)
 {
     size_t colon = s.find(':');
     if (colon == std::string::npos || colon == 0)
         return {"", ""};
-    return {std::string(s, 0, colon), std::string(s, colon + 1)};
+    return {s.substr(0, colon), s.substr(colon + 1)};
 }
 
-Key::Key(const string & s)
+Key::Key(std::string_view s)
 {
     auto ss = split(s);
 
@@ -29,7 +29,12 @@ Key::Key(const string & s)
     key = base64Decode(key);
 }
 
-SecretKey::SecretKey(const string & s)
+std::string Key::to_string() const
+{
+    return name + ":" + base64Encode(key);
+}
+
+SecretKey::SecretKey(std::string_view s)
     : Key(s)
 {
 #if HAVE_SODIUM
@@ -45,7 +50,7 @@ SecretKey::SecretKey(const string & s)
 }
 #endif
 
-std::string SecretKey::signDetached(const std::string & data) const
+std::string SecretKey::signDetached(std::string_view data) const
 {
 #if HAVE_SODIUM
     unsigned char sig[crypto_sign_BYTES];
@@ -69,7 +74,21 @@ PublicKey SecretKey::toPublicKey() const
 #endif
 }
 
-PublicKey::PublicKey(const string & s)
+SecretKey SecretKey::generate(std::string_view name)
+{
+#if HAVE_SODIUM
+    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
+    unsigned char sk[crypto_sign_SECRETKEYBYTES];
+    if (crypto_sign_keypair(pk, sk) != 0)
+        throw Error("key generation failed");
+
+    return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
+#else
+    noSodium();
+#endif
+}
+
+PublicKey::PublicKey(std::string_view s)
     : Key(s)
 {
 #if HAVE_SODIUM
@@ -84,7 +103,7 @@ bool verifyDetached(const std::string & data, const std::string & sig,
 #if HAVE_SODIUM
     auto ss = split(sig);
 
-    auto key = publicKeys.find(ss.first);
+    auto key = publicKeys.find(std::string(ss.first));
     if (key == publicKeys.end()) return false;
 
     auto sig2 = base64Decode(ss.second);
diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh
index 9110af3aa..03f85c103 100644
--- a/src/libstore/crypto.hh
+++ b/src/libstore/crypto.hh
@@ -13,32 +13,40 @@ struct Key
 
     /* Construct Key from a string in the format
        ‘<name>:<key-in-base64>’. */
-    Key(const std::string & s);
+    Key(std::string_view s);
+
+    std::string to_string() const;
 
 protected:
-    Key(const std::string & name, const std::string & key)
-        : name(name), key(key) { }
+    Key(std::string_view name, std::string && key)
+        : name(name), key(std::move(key)) { }
 };
 
 struct PublicKey;
 
 struct SecretKey : Key
 {
-    SecretKey(const std::string & s);
+    SecretKey(std::string_view s);
 
     /* Return a detached signature of the given string. */
-    std::string signDetached(const std::string & s) const;
+    std::string signDetached(std::string_view s) const;
 
     PublicKey toPublicKey() const;
+
+    static SecretKey generate(std::string_view name);
+
+private:
+    SecretKey(std::string_view name, std::string && key)
+        : Key(name, std::move(key)) { }
 };
 
 struct PublicKey : Key
 {
-    PublicKey(const std::string & data);
+    PublicKey(std::string_view data);
 
 private:
-    PublicKey(const std::string & name, const std::string & key)
-        : Key(name, key) { }
+    PublicKey(std::string_view name, std::string && key)
+        : Key(name, std::move(key)) { }
     friend struct SecretKey;
 };
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e1ccece99..e43788bc3 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -19,10 +19,6 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
-#if HAVE_SODIUM
-#include <sodium.h>
-#endif
-
 
 namespace nix_store {
 
@@ -980,18 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
     string secretKeyFile = *i++;
     string publicKeyFile = *i++;
 
-#if HAVE_SODIUM
-    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
-    unsigned char sk[crypto_sign_SECRETKEYBYTES];
-    if (crypto_sign_keypair(pk, sk) != 0)
-        throw Error("key generation failed");
+    auto secretKey = SecretKey::generate(keyName);
 
-    writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES)));
+    writeFile(publicKeyFile, secretKey.toPublicKey().to_string());
     umask(0077);
-    writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES)));
-#else
-    throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
+    writeFile(secretKeyFile, secretKey.to_string());
 }
 
 

From 9374c2baeabe45a22e4b8746dc97f5ce4f030184 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Jan 2021 17:41:16 +0100
Subject: [PATCH 644/882] Add commands for generating secret/public keys

---
 src/nix/hash.cc                         |  5 --
 src/nix/key-convert-secret-to-public.md | 19 ++++++
 src/nix/key-generate-secret.md          | 48 ++++++++++++++
 src/nix/sigs.cc                         | 87 +++++++++++++++++++++++++
 tests/binary-cache.sh                   | 12 ++--
 5 files changed, 160 insertions(+), 11 deletions(-)
 create mode 100644 src/nix/key-convert-secret-to-public.md
 create mode 100644 src/nix/key-generate-secret.md

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 101b67e6a..6fd791f41 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -132,11 +132,6 @@ struct CmdHash : NixMultiCommand
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto rCmdHash = registerCommand<CmdHash>("hash");
diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md
new file mode 100644
index 000000000..3adc18502
--- /dev/null
+++ b/src/nix/key-convert-secret-to-public.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* Convert a secret key to a public key:
+
+  ```console
+  # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \
+    | nix key convert-secret-to-public
+  cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc=
+  ```
+
+# Description
+
+This command reads a Ed25519 secret key from standard input, and
+writes the corresponding public key to standard output. For more
+details, see [nix key generate-secret](./nix3-key-generate-secret.md).
+
+)""
diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md
new file mode 100644
index 000000000..6ff1e1c9b
--- /dev/null
+++ b/src/nix/key-generate-secret.md
@@ -0,0 +1,48 @@
+R""(
+
+# Examples
+
+* Generate a new secret key:
+
+  ```console
+  # nix key generate-secret --key-name cache.example.org-1 > ./secret-key
+  ```
+
+  We can then use this key to sign the closure of the Hello package:
+
+  ```console
+  # nix build nixpkgs#hello
+  # nix store sign-paths --key-file ./secret-key --recursive ./result
+  ```
+
+  Finally, we can verify the store paths using the corresponding
+  public key:
+
+  ```
+  # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result
+  ```
+
+# Description
+
+This command generates a new Ed25519 secret key for signing store
+paths and prints it on standard output. Use `nix key
+convert-secret-to-public` to get the corresponding public key for
+verifying signed store paths.
+
+The mandatory argument `--key-name` specifies a key name (such as
+`cache.example.org-1). It is used to look up keys on the client when
+it verifies signatures. It can be anything, but it’s suggested to use
+the host name of your cache (e.g.  `cache.example.org`) with a suffix
+denoting the number of the key (to be incremented every time you need
+to revoke a key).
+
+# Format
+
+Both secret and public keys are represented as the key name followed
+by a base-64 encoding of the Ed25519 key data, e.g.
+
+```
+cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw==
+```
+
+)""
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 37b8a6712..b2e598ad5 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -141,3 +141,90 @@ struct CmdSignPaths : StorePathsCommand
 };
 
 static auto rCmdSignPaths = registerCommand2<CmdSignPaths>({"store", "sign-paths"});
+
+#if HAVE_SODIUM
+struct CmdKeyGenerateSecret : Command
+{
+    std::optional<std::string> keyName;
+
+    CmdKeyGenerateSecret()
+    {
+        addFlag({
+            .longName = "key-name",
+            .description = "identifier of the key (e.g. `cache.example.org-1`)",
+            .labels = {"name"},
+            .handler = {&keyName},
+        });
+    }
+
+    std::string description() override
+    {
+        return "generate a secret key for signing store paths";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "key-generate-secret.md"
+          ;
+    }
+
+    void run() override
+    {
+        if (!keyName)
+            throw UsageError("required argument '--key-name' is missing");
+
+        std::cout << SecretKey::generate(*keyName).to_string();
+    }
+};
+
+struct CmdKeyConvertSecretToPublic : Command
+{
+    std::string description() override
+    {
+        return "generate a public key for verifying store paths from a secret key read from standard input";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "key-convert-secret-to-public.md"
+          ;
+    }
+
+    void run() override
+    {
+        SecretKey secretKey(drainFD(STDIN_FILENO));
+        std::cout << secretKey.toPublicKey().to_string();
+    }
+};
+
+struct CmdKey : NixMultiCommand
+{
+    CmdKey()
+        : MultiCommand({
+                {"generate-secret", []() { return make_ref<CmdKeyGenerateSecret>(); }},
+                {"convert-secret-to-public", []() { return make_ref<CmdKeyConvertSecretToPublic>(); }},
+            })
+    {
+    }
+
+    std::string description() override
+    {
+        return "generate and convert Nix signing keys";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix flake' requires a sub-command.");
+        settings.requireExperimentalFeature("flakes");
+        command->second->prepare();
+        command->second->run();
+    }
+};
+
+static auto rCmdKey = registerCommand<CmdKey>("key");
+#endif
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 92ed36225..1a06404ed 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -131,14 +131,14 @@ if [ -n "$HAVE_SODIUM" ]; then
 clearCache
 clearCacheCache
 
-declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 ))
-publicKey="$(cat $TEST_ROOT/pk1)"
+nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1
+publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1)
 
-res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2))
-badKey="$(cat $TEST_ROOT/pk2)"
+nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2
+badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2)
 
-res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
-otherKey="$(cat $TEST_ROOT/pk3)"
+nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3
+otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3)
 
 _NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 

From 0df69d96e02ce4c9e17bd33333c5d78313341dd3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Jan 2021 17:56:53 +0100
Subject: [PATCH 645/882] Make sodium a required dependency

---
 Makefile.config.in      |  1 -
 configure.ac            |  6 +-----
 perl/Makefile.config.in |  1 -
 perl/configure.ac       |  6 +-----
 perl/lib/Nix/Store.xs   | 10 ----------
 src/libmain/shared.cc   |  6 ------
 src/libstore/crypto.cc  | 29 -----------------------------
 src/nix/sigs.cc         |  2 --
 tests/binary-cache.sh   |  4 ----
 tests/common.sh.in      |  1 -
 10 files changed, 2 insertions(+), 64 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 3845b3be0..d1e59e4e7 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -10,7 +10,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_SECCOMP = @HAVE_SECCOMP@
-HAVE_SODIUM = @HAVE_SODIUM@
 LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
diff --git a/configure.ac b/configure.ac
index c1bfc9b53..2047ed8d2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -203,11 +203,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
 ])
 
 # Look for libsodium, an optional dependency.
-PKG_CHECK_MODULES([SODIUM], [libsodium],
-  [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])
-   CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"
-   have_sodium=1], [have_sodium=])
-AC_SUBST(HAVE_SODIUM, [$have_sodium])
+PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in
index c87d4817e..eccfbd9f6 100644
--- a/perl/Makefile.config.in
+++ b/perl/Makefile.config.in
@@ -2,7 +2,6 @@ CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-HAVE_SODIUM = @HAVE_SODIUM@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
diff --git a/perl/configure.ac b/perl/configure.ac
index 255744afd..85183c005 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -40,11 +40,7 @@ AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
 # Look for libsodium, an optional dependency.
-PKG_CHECK_MODULES([SODIUM], [libsodium],
-  [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])
-   CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"
-   have_sodium=1], [have_sodium=])
-AC_SUBST(HAVE_SODIUM, [$have_sodium])
+PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Check for the required Perl dependencies (DBI and DBD::SQLite).
 perlFlags="-I$perllibdir"
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 9e3b7d389..ad9042a2a 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -14,9 +14,7 @@
 #include "util.hh"
 #include "crypto.hh"
 
-#if HAVE_SODIUM
 #include <sodium.h>
-#endif
 
 
 using namespace nix;
@@ -239,12 +237,8 @@ SV * convertHash(char * algo, char * s, int toBase32)
 SV * signString(char * secretKey_, char * msg)
     PPCODE:
         try {
-#if HAVE_SODIUM
             auto sig = SecretKey(secretKey_).signDetached(msg);
             XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size())));
-#else
-            throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -253,7 +247,6 @@ SV * signString(char * secretKey_, char * msg)
 int checkSignature(SV * publicKey_, SV * sig_, char * msg)
     CODE:
         try {
-#if HAVE_SODIUM
             STRLEN publicKeyLen;
             unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen);
             if (publicKeyLen != crypto_sign_PUBLICKEYBYTES)
@@ -265,9 +258,6 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
                 throw Error("signature is not valid");
 
             RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0;
-#else
-            throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
         } catch (Error & e) {
             croak("%s", e.what());
         }
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index e9f067e35..6751a3744 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -18,9 +18,7 @@
 
 #include <openssl/crypto.h>
 
-#if HAVE_SODIUM
 #include <sodium.h>
-#endif
 
 
 namespace nix {
@@ -130,10 +128,8 @@ void initNix()
     CRYPTO_set_locking_callback(opensslLockCallback);
 #endif
 
-#if HAVE_SODIUM
     if (sodium_init() == -1)
         throw Error("could not initialise libsodium");
-#endif
 
     loadConfFile();
 
@@ -283,9 +279,7 @@ void printVersion(const string & programName)
 #if HAVE_BOEHMGC
         cfg.push_back("gc");
 #endif
-#if HAVE_SODIUM
         cfg.push_back("signed-caches");
-#endif
         std::cout << "System type: " << settings.thisSystem << "\n";
         std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 135ced277..1027469c9 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -2,9 +2,7 @@
 #include "util.hh"
 #include "globals.hh"
 
-#if HAVE_SODIUM
 #include <sodium.h>
-#endif
 
 namespace nix {
 
@@ -37,70 +35,46 @@ std::string Key::to_string() const
 SecretKey::SecretKey(std::string_view s)
     : Key(s)
 {
-#if HAVE_SODIUM
     if (key.size() != crypto_sign_SECRETKEYBYTES)
         throw Error("secret key is not valid");
-#endif
 }
 
-#if !HAVE_SODIUM
-[[noreturn]] static void noSodium()
-{
-    throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-}
-#endif
-
 std::string SecretKey::signDetached(std::string_view data) const
 {
-#if HAVE_SODIUM
     unsigned char sig[crypto_sign_BYTES];
     unsigned long long sigLen;
     crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(),
         (unsigned char *) key.data());
     return name + ":" + base64Encode(std::string((char *) sig, sigLen));
-#else
-    noSodium();
-#endif
 }
 
 PublicKey SecretKey::toPublicKey() const
 {
-#if HAVE_SODIUM
     unsigned char pk[crypto_sign_PUBLICKEYBYTES];
     crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data());
     return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES));
-#else
-    noSodium();
-#endif
 }
 
 SecretKey SecretKey::generate(std::string_view name)
 {
-#if HAVE_SODIUM
     unsigned char pk[crypto_sign_PUBLICKEYBYTES];
     unsigned char sk[crypto_sign_SECRETKEYBYTES];
     if (crypto_sign_keypair(pk, sk) != 0)
         throw Error("key generation failed");
 
     return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
-#else
-    noSodium();
-#endif
 }
 
 PublicKey::PublicKey(std::string_view s)
     : Key(s)
 {
-#if HAVE_SODIUM
     if (key.size() != crypto_sign_PUBLICKEYBYTES)
         throw Error("public key is not valid");
-#endif
 }
 
 bool verifyDetached(const std::string & data, const std::string & sig,
     const PublicKeys & publicKeys)
 {
-#if HAVE_SODIUM
     auto ss = split(sig);
 
     auto key = publicKeys.find(std::string(ss.first));
@@ -113,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig,
     return crypto_sign_verify_detached((unsigned char *) sig2.data(),
         (unsigned char *) data.data(), data.size(),
         (unsigned char *) key->second.key.data()) == 0;
-#else
-    noSodium();
-#endif
 }
 
 PublicKeys getDefaultPublicKeys()
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index b2e598ad5..14e2c9761 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -142,7 +142,6 @@ struct CmdSignPaths : StorePathsCommand
 
 static auto rCmdSignPaths = registerCommand2<CmdSignPaths>({"store", "sign-paths"});
 
-#if HAVE_SODIUM
 struct CmdKeyGenerateSecret : Command
 {
     std::optional<std::string> keyName;
@@ -227,4 +226,3 @@ struct CmdKey : NixMultiCommand
 };
 
 static auto rCmdKey = registerCommand<CmdKey>("key");
-#endif
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 1a06404ed..355a37d97 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -125,8 +125,6 @@ grep -q "copying path.*input-0" $TEST_ROOT/log
 grep -q "copying path.*top" $TEST_ROOT/log
 
 
-if [ -n "$HAVE_SODIUM" ]; then
-
 # Create a signed binary cache.
 clearCache
 clearCacheCache
@@ -181,8 +179,6 @@ clearCacheCache
 
 nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
 
-fi # HAVE_LIBSODIUM
-
 
 unset _NIX_FORCE_HTTP
 
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 5e00d64f1..5489c0c44 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -34,7 +34,6 @@ coreutils=@coreutils@
 export dot=@dot@
 export SHELL="@bash@"
 export PAGER=cat
-export HAVE_SODIUM="@HAVE_SODIUM@"
 export busybox="@sandbox_shell@"
 
 export version=@PACKAGE_VERSION@

From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Jan 2021 21:51:46 +0100
Subject: [PATCH 646/882] Add 'nix store prefetch-{file,tarball}'

These replace nix-prefetch-url and nix-prefetch-url --unpack,
respectively.
---
 src/libstore/filetransfer.hh             |   2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc | 232 ---------------
 src/nix/local.mk                         |   1 -
 src/nix/prefetch.cc                      | 352 +++++++++++++++++++++++
 src/nix/store-prefetch-file.md           |  32 +++
 src/nix/store-prefetch-tarball.md        |  31 ++
 6 files changed, 416 insertions(+), 234 deletions(-)
 delete mode 100644 src/nix-prefetch-url/nix-prefetch-url.cc
 create mode 100644 src/nix/prefetch.cc
 create mode 100644 src/nix/store-prefetch-file.md
 create mode 100644 src/nix/store-prefetch-tarball.md

diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index afc7e7aa6..45d9ccf89 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -63,7 +63,7 @@ struct FileTransferRequest
     std::string mimeType;
     std::function<void(std::string_view data)> dataCallback;
 
-    FileTransferRequest(const std::string & uri)
+    FileTransferRequest(std::string_view uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
     std::string verb()
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
deleted file mode 100644
index 3bdee55a7..000000000
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ /dev/null
@@ -1,232 +0,0 @@
-#include "hash.hh"
-#include "shared.hh"
-#include "filetransfer.hh"
-#include "store-api.hh"
-#include "eval.hh"
-#include "eval-inline.hh"
-#include "common-eval-args.hh"
-#include "attr-path.hh"
-#include "finally.hh"
-#include "../nix/legacy.hh"
-#include "progress-bar.hh"
-#include "tarfile.hh"
-
-#include <iostream>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-using namespace nix;
-
-
-/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of
-   mirrors defined in Nixpkgs. */
-string resolveMirrorUri(EvalState & state, string uri)
-{
-    if (string(uri, 0, 9) != "mirror://") return uri;
-
-    string s(uri, 9);
-    auto p = s.find('/');
-    if (p == string::npos) throw Error("invalid mirror URI");
-    string mirrorName(s, 0, p);
-
-    Value vMirrors;
-    state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors);
-    state.forceAttrs(vMirrors);
-
-    auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
-    if (mirrorList == vMirrors.attrs->end())
-        throw Error("unknown mirror name '%1%'", mirrorName);
-    state.forceList(*mirrorList->value);
-
-    if (mirrorList->value->listSize() < 1)
-        throw Error("mirror URI '%1%' did not expand to anything", uri);
-
-    string mirror = state.forceString(*mirrorList->value->listElems()[0]);
-    return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
-}
-
-
-static int main_nix_prefetch_url(int argc, char * * argv)
-{
-    {
-        HashType ht = htSHA256;
-        std::vector<string> args;
-        bool printPath = getEnv("PRINT_PATH") == "1";
-        bool fromExpr = false;
-        string attrPath;
-        bool unpack = false;
-        bool executable = false;
-        string name;
-
-        struct MyArgs : LegacyArgs, MixEvalArgs
-        {
-            using LegacyArgs::LegacyArgs;
-        };
-
-        MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
-            if (*arg == "--help")
-                showManPage("nix-prefetch-url");
-            else if (*arg == "--version")
-                printVersion("nix-prefetch-url");
-            else if (*arg == "--type") {
-                string s = getArg(*arg, arg, end);
-                ht = parseHashType(s);
-            }
-            else if (*arg == "--print-path")
-                printPath = true;
-            else if (*arg == "--attr" || *arg == "-A") {
-                fromExpr = true;
-                attrPath = getArg(*arg, arg, end);
-            }
-            else if (*arg == "--unpack")
-                unpack = true;
-            else if (*arg == "--executable")
-                executable = true;
-            else if (*arg == "--name")
-                name = getArg(*arg, arg, end);
-            else if (*arg != "" && arg->at(0) == '-')
-                return false;
-            else
-                args.push_back(*arg);
-            return true;
-        });
-
-        myArgs.parseCmdline(argvToStrings(argc, argv));
-
-        initPlugins();
-
-        if (args.size() > 2)
-            throw UsageError("too many arguments");
-
-        Finally f([]() { stopProgressBar(); });
-
-        if (isatty(STDERR_FILENO))
-          startProgressBar();
-
-        auto store = openStore();
-        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
-
-        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
-
-        /* If -A is given, get the URI from the specified Nix
-           expression. */
-        string uri;
-        if (!fromExpr) {
-            if (args.empty())
-                throw UsageError("you must specify a URI");
-            uri = args[0];
-        } else {
-            Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
-            Value vRoot;
-            state->evalFile(path, vRoot);
-            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
-            state->forceAttrs(v);
-
-            /* Extract the URI. */
-            auto attr = v.attrs->find(state->symbols.create("urls"));
-            if (attr == v.attrs->end())
-                throw Error("attribute set does not contain a 'urls' attribute");
-            state->forceList(*attr->value);
-            if (attr->value->listSize() < 1)
-                throw Error("'urls' list is empty");
-            uri = state->forceString(*attr->value->listElems()[0]);
-
-            /* Extract the hash mode. */
-            attr = v.attrs->find(state->symbols.create("outputHashMode"));
-            if (attr == v.attrs->end())
-                printInfo("warning: this does not look like a fetchurl call");
-            else
-                unpack = state->forceString(*attr->value) == "recursive";
-
-            /* Extract the name. */
-            if (name.empty()) {
-                attr = v.attrs->find(state->symbols.create("name"));
-                if (attr != v.attrs->end())
-                    name = state->forceString(*attr->value);
-            }
-        }
-
-        /* Figure out a name in the Nix store. */
-        if (name.empty())
-            name = baseNameOf(uri);
-        if (name.empty())
-            throw Error("cannot figure out file name for '%1%'", uri);
-
-        /* If an expected hash is given, the file may already exist in
-           the store. */
-        std::optional<Hash> expectedHash;
-        Hash hash(ht);
-        std::optional<StorePath> storePath;
-        if (args.size() == 2) {
-            expectedHash = Hash::parseAny(args[1], ht);
-            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
-            if (store->isValidPath(*storePath))
-                hash = *expectedHash;
-            else
-                storePath.reset();
-        }
-
-        if (!storePath) {
-
-            auto actualUri = resolveMirrorUri(*state, uri);
-
-            AutoDelete tmpDir(createTempDir(), true);
-            Path tmpFile = (Path) tmpDir + "/tmp";
-
-            /* Download the file. */
-            {
-                auto mode = 0600;
-                if (executable)
-                    mode = 0700;
-
-                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
-                if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
-
-                FdSink sink(fd.get());
-
-                FileTransferRequest req(actualUri);
-                req.decompress = false;
-                getFileTransfer()->download(std::move(req), sink);
-            }
-
-            /* Optionally unpack the file. */
-            if (unpack) {
-                printInfo("unpacking...");
-                Path unpacked = (Path) tmpDir + "/unpacked";
-                createDirs(unpacked);
-                unpackTarfile(tmpFile, unpacked);
-
-                /* If the archive unpacks to a single file/directory, then use
-                   that as the top-level. */
-                auto entries = readDirectory(unpacked);
-                if (entries.size() == 1)
-                    tmpFile = unpacked + "/" + entries[0].name;
-                else
-                    tmpFile = unpacked;
-            }
-
-            const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
-            auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
-            storePath = info.path;
-            assert(info.ca);
-            hash = getContentAddressHash(*info.ca);
-        }
-
-        stopProgressBar();
-
-        if (!printPath)
-            printInfo("path is '%s'", store->printStorePath(*storePath));
-
-        std::cout << printHash16or32(hash) << std::endl;
-        if (printPath)
-            std::cout << store->printStorePath(*storePath) << std::endl;
-
-        return 0;
-    }
-}
-
-static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);
diff --git a/src/nix/local.mk b/src/nix/local.mk
index f37b73384..23c08fc86 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -12,7 +12,6 @@ nix_SOURCES := \
   $(wildcard src/nix-daemon/*.cc) \
   $(wildcard src/nix-env/*.cc) \
   $(wildcard src/nix-instantiate/*.cc) \
-  $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
 nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
new file mode 100644
index 000000000..969299489
--- /dev/null
+++ b/src/nix/prefetch.cc
@@ -0,0 +1,352 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "filetransfer.hh"
+#include "finally.hh"
+#include "progress-bar.hh"
+#include "tarfile.hh"
+#include "attr-path.hh"
+#include "eval-inline.hh"
+#include "legacy.hh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nix;
+
+/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of
+   mirrors defined in Nixpkgs. */
+string resolveMirrorUrl(EvalState & state, string url)
+{
+    if (url.substr(0, 9) != "mirror://") return url;
+
+    std::string s(url, 9);
+    auto p = s.find('/');
+    if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url);
+    std::string mirrorName(s, 0, p);
+
+    Value vMirrors;
+    // FIXME: use nixpkgs flake
+    state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors);
+    state.forceAttrs(vMirrors);
+
+    auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
+    if (mirrorList == vMirrors.attrs->end())
+        throw Error("unknown mirror name '%s'", mirrorName);
+    state.forceList(*mirrorList->value);
+
+    if (mirrorList->value->listSize() < 1)
+        throw Error("mirror URL '%s' did not expand to anything", url);
+
+    auto mirror = state.forceString(*mirrorList->value->listElems()[0]);
+    return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
+}
+
+std::tuple<StorePath, Hash> prefetchFile(
+    ref<Store> store,
+    std::string_view url,
+    std::optional<std::string> name,
+    HashType hashType,
+    std::optional<Hash> expectedHash,
+    bool unpack,
+    bool executable)
+{
+    auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+
+    /* Figure out a name in the Nix store. */
+    if (!name) {
+        name = baseNameOf(url);
+        if (name->empty())
+            throw Error("cannot figure out file name for '%s'", url);
+    }
+
+    std::optional<StorePath> storePath;
+    std::optional<Hash> hash;
+
+    /* If an expected hash is given, the file may already exist in
+       the store. */
+    if (expectedHash) {
+        hashType = expectedHash->type;
+        storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name);
+        if (store->isValidPath(*storePath))
+            hash = expectedHash;
+        else
+            storePath.reset();
+    }
+
+    if (!storePath) {
+
+        AutoDelete tmpDir(createTempDir(), true);
+        Path tmpFile = (Path) tmpDir + "/tmp";
+
+        /* Download the file. */
+        {
+            auto mode = 0600;
+            if (executable)
+                mode = 0700;
+
+            AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
+            if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
+
+            FdSink sink(fd.get());
+
+            FileTransferRequest req(url);
+            req.decompress = false;
+            getFileTransfer()->download(std::move(req), sink);
+        }
+
+        /* Optionally unpack the file. */
+        if (unpack) {
+            Activity act(*logger, lvlChatty, actUnknown,
+                fmt("unpacking '%s'", url));
+            Path unpacked = (Path) tmpDir + "/unpacked";
+            createDirs(unpacked);
+            unpackTarfile(tmpFile, unpacked);
+
+            /* If the archive unpacks to a single file/directory, then use
+               that as the top-level. */
+            auto entries = readDirectory(unpacked);
+            if (entries.size() == 1)
+                tmpFile = unpacked + "/" + entries[0].name;
+            else
+                tmpFile = unpacked;
+        }
+
+        Activity act(*logger, lvlChatty, actUnknown,
+            fmt("adding '%s' to the store", url));
+
+        auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
+        storePath = info.path;
+        assert(info.ca);
+        hash = getContentAddressHash(*info.ca);
+    }
+
+    return {storePath.value(), hash.value()};
+}
+
+static int main_nix_prefetch_url(int argc, char * * argv)
+{
+    {
+        HashType ht = htSHA256;
+        std::vector<string> args;
+        bool printPath = getEnv("PRINT_PATH") == "1";
+        bool fromExpr = false;
+        string attrPath;
+        bool unpack = false;
+        bool executable = false;
+        std::optional<std::string> name;
+
+        struct MyArgs : LegacyArgs, MixEvalArgs
+        {
+            using LegacyArgs::LegacyArgs;
+        };
+
+        MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help")
+                showManPage("nix-prefetch-url");
+            else if (*arg == "--version")
+                printVersion("nix-prefetch-url");
+            else if (*arg == "--type") {
+                string s = getArg(*arg, arg, end);
+                ht = parseHashType(s);
+            }
+            else if (*arg == "--print-path")
+                printPath = true;
+            else if (*arg == "--attr" || *arg == "-A") {
+                fromExpr = true;
+                attrPath = getArg(*arg, arg, end);
+            }
+            else if (*arg == "--unpack")
+                unpack = true;
+            else if (*arg == "--executable")
+                executable = true;
+            else if (*arg == "--name")
+                name = getArg(*arg, arg, end);
+            else if (*arg != "" && arg->at(0) == '-')
+                return false;
+            else
+                args.push_back(*arg);
+            return true;
+        });
+
+        myArgs.parseCmdline(argvToStrings(argc, argv));
+
+        initPlugins();
+
+        if (args.size() > 2)
+            throw UsageError("too many arguments");
+
+        Finally f([]() { stopProgressBar(); });
+
+        if (isatty(STDERR_FILENO))
+          startProgressBar();
+
+        auto store = openStore();
+        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
+
+        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
+
+        /* If -A is given, get the URL from the specified Nix
+           expression. */
+        string url;
+        if (!fromExpr) {
+            if (args.empty())
+                throw UsageError("you must specify a URL");
+            url = args[0];
+        } else {
+            Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
+            Value vRoot;
+            state->evalFile(path, vRoot);
+            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
+            state->forceAttrs(v);
+
+            /* Extract the URL. */
+            auto attr = v.attrs->find(state->symbols.create("urls"));
+            if (attr == v.attrs->end())
+                throw Error("attribute set does not contain a 'urls' attribute");
+            state->forceList(*attr->value);
+            if (attr->value->listSize() < 1)
+                throw Error("'urls' list is empty");
+            url = state->forceString(*attr->value->listElems()[0]);
+
+            /* Extract the hash mode. */
+            attr = v.attrs->find(state->symbols.create("outputHashMode"));
+            if (attr == v.attrs->end())
+                printInfo("warning: this does not look like a fetchurl call");
+            else
+                unpack = state->forceString(*attr->value) == "recursive";
+
+            /* Extract the name. */
+            if (!name) {
+                attr = v.attrs->find(state->symbols.create("name"));
+                if (attr != v.attrs->end())
+                    name = state->forceString(*attr->value);
+            }
+        }
+
+        std::optional<Hash> expectedHash;
+        if (args.size() == 2)
+            expectedHash = Hash::parseAny(args[1], ht);
+
+        auto [storePath, hash] = prefetchFile(
+            store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable);
+
+        stopProgressBar();
+
+        if (!printPath)
+            printInfo("path is '%s'", store->printStorePath(storePath));
+
+        std::cout << printHash16or32(hash) << std::endl;
+        if (printPath)
+            std::cout << store->printStorePath(storePath) << std::endl;
+
+        return 0;
+    }
+}
+
+static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);
+
+struct CmdStorePrefetch : StoreCommand, MixJSON
+{
+    std::string url;
+    bool executable = false;
+    bool unpack;
+    std::optional<std::string> name;
+    HashType hashType = htSHA256;
+    std::optional<Hash> expectedHash;
+
+    CmdStorePrefetch(bool unpack)
+        : unpack(unpack)
+    {
+        addFlag({
+            .longName = "name",
+            .description = "store path name",
+            .labels = {"name"},
+            .handler = {&name}
+        });
+
+        addFlag({
+            .longName = "expected-hash",
+            .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file",
+            .labels = {"hash"},
+            .handler = {[&](std::string s) {
+                expectedHash = Hash::parseAny(s, hashType);
+            }}
+        });
+
+        addFlag(Flag::mkHashTypeFlag("hash-type", &hashType));
+
+        expectArg("url", &url);
+    }
+
+    Category category() override { return catUtility; }
+
+    void run(ref<Store> store) override
+    {
+        auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable);
+
+        if (json) {
+            auto res = nlohmann::json::object();
+            res["storePath"] = store->printStorePath(storePath);
+            res["hash"] = hash.to_string(SRI, true);
+            logger->cout(res.dump());
+        } else {
+            notice("Downloaded '%s' to '%s' (hash '%s').",
+                url,
+                store->printStorePath(storePath),
+                hash.to_string(SRI, true));
+        }
+    }
+};
+
+struct CmdStorePrefetchFile : CmdStorePrefetch
+{
+    CmdStorePrefetchFile()
+        : CmdStorePrefetch(false)
+    {
+        name = "source";
+
+        addFlag({
+            .longName = "executable",
+            .description = "make the resulting file executable",
+            .handler = {&executable, true},
+        });
+    }
+
+    std::string description() override
+    {
+        return "download a file into the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-prefetch-file.md"
+          ;
+    }
+};
+
+static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"});
+
+struct CmdStorePrefetchTarball : CmdStorePrefetch
+{
+    CmdStorePrefetchTarball()
+        : CmdStorePrefetch(true)
+    {
+        name = "source";
+    }
+
+    std::string description() override
+    {
+        return "download and unpack a tarball into the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-prefetch-tarball.md"
+          ;
+    }
+};
+
+static auto rCmdStorePrefetchTarball = registerCommand2<CmdStorePrefetchTarball>({"store", "prefetch-tarball"});
diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md
new file mode 100644
index 000000000..1663b847b
--- /dev/null
+++ b/src/nix/store-prefetch-file.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Download a file to the Nix store:
+
+  ```console
+  # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz
+  Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to
+  '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash
+  'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s=').
+  ```
+
+* Download a file and get the SHA-512 hash:
+
+  ```console
+  # nix store prefetch-file --json --hash-type sha512 \
+      https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \
+    | jq -r .hash
+  sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA==
+  ```
+
+# Description
+
+This command downloads the file *url* to the Nix store. It prints out
+the resulting store path and the cryptographic hash of the contents of
+the file.
+
+The name component of the store path defaults to the last component of
+*url*, but this can be overriden using `--name`.
+
+)""
diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md
new file mode 100644
index 000000000..535d7e022
--- /dev/null
+++ b/src/nix/store-prefetch-tarball.md
@@ -0,0 +1,31 @@
+R""(
+
+# Examples
+
+* Download a tarball and unpack it:
+
+  ```console
+  # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz
+  Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz'
+  to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash
+  'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=').
+  ```
+
+* Download a tarball and unpack it, unless it already exists in the
+  Nix store:
+
+  ```console
+  # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \
+      --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=
+  ```
+
+# Description
+
+This command downloads a tarball or zip file from *url*, unpacks it,
+and adds the unpacked tree to the Nix store. It prints out the
+resulting store path and the NAR hash of that store path.
+
+The name component of the store path defaults to `source`, but this
+can be overriden using `--name`.
+
+)""

From 93f1678ec60bcacfcc857f361b5f63e37c498eb4 Mon Sep 17 00:00:00 2001
From: Danila Fedorin <danila.fedorin@gmail.com>
Date: Fri, 8 Jan 2021 01:53:57 +0000
Subject: [PATCH 647/882] Allow Flake inputs to accept boolean and integer
 attributes

I believe that this makes it possible to do things like
Git inputs with submodules, but it also likely applies
to other input types from libfetchers.
---
 src/libexpr/flake/flake.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 4f021570c..41c93bcaa 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -120,11 +120,16 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
-                if (attr.value->type() == nString)
+                if (attr.value->type() == nString) {
                     attrs.emplace(attr.name, attr.value->string.s);
-                else
-                    throw TypeError("flake input attribute '%s' is %s while a string is expected",
+                } else if (attr.value->type() == nBool) {
+                    attrs.emplace(attr.name, Explicit<bool>{ attr.value->boolean });
+                } else if (attr.value->type() == nInt) {
+                    attrs.emplace(attr.name, attr.value->integer);
+                } else {
+                    throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected",
                         attr.name, showType(*attr.value));
+                }
             }
         } catch (Error & e) {
             e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));

From ba0f841a078402f95cf93693c3749743c3ab6246 Mon Sep 17 00:00:00 2001
From: Danila Fedorin <danila.fedorin@gmail.com>
Date: Fri, 8 Jan 2021 03:13:42 +0000
Subject: [PATCH 648/882] Use switch statement instead of sequence of ifs

---
 src/libexpr/flake/flake.cc | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 41c93bcaa..9f1e4063f 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -120,15 +120,19 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
-                if (attr.value->type() == nString) {
-                    attrs.emplace(attr.name, attr.value->string.s);
-                } else if (attr.value->type() == nBool) {
-                    attrs.emplace(attr.name, Explicit<bool>{ attr.value->boolean });
-                } else if (attr.value->type() == nInt) {
-                    attrs.emplace(attr.name, attr.value->integer);
-                } else {
-                    throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected",
-                        attr.name, showType(*attr.value));
+                switch (attr.value->type()) {
+                    case nString:
+                        attrs.emplace(attr.name, attr.value->string.s);
+                        break;
+                    case nBool:
+                        attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean });
+                        break;
+                    case nInt:
+                        attrs.emplace(attr.name, attr.value->integer);
+                        break;
+                    default:
+                        throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected",
+                            attr.name, showType(*attr.value));
                 }
             }
         } catch (Error & e) {

From 48a9be2aabf6620ceb00caf7c4c917e4e0a81446 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Jan 2021 10:44:55 +0100
Subject: [PATCH 649/882] Remove mkIntFlag

---
 src/libutil/args.hh | 26 +++++++++++++++-----------
 src/nix/verify.cc   | 10 +++++++++-
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 6ed541a32..3e84ac64a 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -68,8 +68,12 @@ protected:
             , arity(ArityAny)
         { }
 
-        template<class T>
-        Handler(T * dest)
+        Handler(std::string * dest)
+            : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+            , arity(1)
+        { }
+
+        Handler(std::optional<std::string> * dest)
             : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
             , arity(1)
         { }
@@ -79,6 +83,15 @@ protected:
             : fun([=](std::vector<std::string> ss) { *dest = val; })
             , arity(0)
         { }
+
+        template<class I>
+        Handler(I * dest)
+            : fun([=](std::vector<std::string> ss) {
+                if (!string2Int(ss[0], *dest))
+                    throw UsageError("'%s' is not an integer", ss[0]);
+              })
+            , arity(1)
+        { }
     };
 
     /* Flags. */
@@ -161,15 +174,6 @@ public:
         });
     }
 
-    template<class I>
-    void mkIntFlag(char shortName, const std::string & longName,
-        const std::string & description, I * dest)
-    {
-        mkFlag<I>(shortName, longName, description, [=](I n) {
-            *dest = n;
-        });
-    }
-
     template<class I>
     void mkFlag(char shortName, const std::string & longName,
         const std::string & description, std::function<void(I)> fun)
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 16d42349f..620109aac 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -20,6 +20,7 @@ struct CmdVerify : StorePathsCommand
     {
         mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
         mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
+
         addFlag({
             .longName = "substituter",
             .shortName = 's',
@@ -27,7 +28,14 @@ struct CmdVerify : StorePathsCommand
             .labels = {"store-uri"},
             .handler = {[&](std::string s) { substituterUris.push_back(s); }}
         });
-        mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
+
+        addFlag({
+            .longName = "sigs-needed",
+            .shortName = 'n',
+            .description = "require that each path has at least N valid signatures",
+            .labels = {"n"},
+            .handler = {&sigsNeeded}
+        });
     }
 
     std::string description() override

From 1d4954e73e389d319416bf29e949b4b1cbc9ebd9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Jan 2021 11:40:36 +0100
Subject: [PATCH 650/882] Remove mkFlag integer specialisation

---
 src/libmain/shared.cc | 16 +++++++++++++---
 src/libutil/args.hh   | 18 ------------------
 2 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 6751a3744..223020378 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -211,9 +211,19 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     });
 
     auto intSettingAlias = [&](char shortName, const std::string & longName,
-        const std::string & description, const std::string & dest) {
-        mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
-            settings.set(dest, std::to_string(n));
+        const std::string & description, const std::string & dest)
+    {
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {"n"},
+            .handler = {[=](std::string s) {
+                unsigned int n;
+                if (!string2Int(s, n))
+                    throw UsageError("'%s' is not an integer", s);
+                settings.set(dest, std::to_string(n));
+            }}
         });
     };
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 3e84ac64a..c54b0efaf 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -174,24 +174,6 @@ public:
         });
     }
 
-    template<class I>
-    void mkFlag(char shortName, const std::string & longName,
-        const std::string & description, std::function<void(I)> fun)
-    {
-        addFlag({
-            .longName = longName,
-            .shortName = shortName,
-            .description = description,
-            .labels = {"N"},
-            .handler = {[=](std::string s) {
-                I n;
-                if (!string2Int(s, n))
-                    throw UsageError("flag '--%s' requires a integer argument", longName);
-                fun(n);
-            }}
-        });
-    }
-
     void expectArgs(ExpectedArg && arg)
     {
         expectedArgs.emplace_back(std::move(arg));

From 29a445840a4f01dfb1533806f8dfc28f7dc4bee9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Jan 2021 11:42:44 +0100
Subject: [PATCH 651/882] Remove unused mkFlag1

---
 src/libutil/args.hh | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index c54b0efaf..62b9516d8 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -143,19 +143,6 @@ public:
     /* Helper functions for constructing flags / positional
        arguments. */
 
-    void mkFlag1(char shortName, const std::string & longName,
-        const std::string & label, const std::string & description,
-        std::function<void(std::string)> fun)
-    {
-        addFlag({
-            .longName = longName,
-            .shortName = shortName,
-            .description = description,
-            .labels = {label},
-            .handler = {[=](std::string s) { fun(s); }}
-        });
-    }
-
     void mkFlag(char shortName, const std::string & name,
         const std::string & description, bool * dest)
     {

From 6548b89cc4eb214cb4632fd4332c610f2d1f0a9d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Jan 2021 12:22:21 +0100
Subject: [PATCH 652/882] string2Int(): Return std::optional

---
 src/libexpr/attr-path.cc              | 14 ++++++--------
 src/libexpr/get-drvs.cc               |  8 ++++----
 src/libfetchers/path.cc               |  6 +++---
 src/libmain/shared.cc                 |  6 +++---
 src/libmain/shared.hh                 |  7 +++----
 src/libstore/build/derivation-goal.cc |  4 +---
 src/libstore/globals.cc               |  8 ++++++--
 src/libstore/local-store.cc           |  4 +++-
 src/libstore/names.cc                 | 12 ++++++------
 src/libstore/nar-info.cc              |  8 ++++++--
 src/libstore/profiles.cc              | 11 +++++------
 src/libstore/store-api.cc             | 13 +++++++------
 src/libutil/args.hh                   |  4 +++-
 src/libutil/config.cc                 |  4 +++-
 src/libutil/util.hh                   | 18 ++++++++++++------
 src/nix-env/nix-env.cc                | 19 +++++++++----------
 src/nix/profile.cc                    |  5 ++---
 17 files changed, 82 insertions(+), 69 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 2d37dcb7e..9dd557205 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -52,9 +52,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
     for (auto & attr : tokens) {
 
         /* Is i an index (integer) or a normal attribute name? */
-        enum { apAttr, apIndex } apType = apAttr;
-        unsigned int attrIndex;
-        if (string2Int(attr, attrIndex)) apType = apIndex;
+        auto attrIndex = string2Int<unsigned int>(attr);
 
         /* Evaluate the expression. */
         Value * vNew = state.allocValue();
@@ -65,7 +63,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         /* It should evaluate to either a set or an expression,
            according to what is specified in the attrPath. */
 
-        if (apType == apAttr) {
+        if (!attrIndex) {
 
             if (v->type() != nAttrs)
                 throw TypeError(
@@ -82,17 +80,17 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
             pos = *a->pos;
         }
 
-        else if (apType == apIndex) {
+        else {
 
             if (!v->isList())
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a list but is %2%",
                     attrPath,
                     showType(*v));
-            if (attrIndex >= v->listSize())
-                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
+            if (*attrIndex >= v->listSize())
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
 
-            v = v->listElems()[attrIndex];
+            v = v->listElems()[*attrIndex];
             pos = noPos;
         }
 
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 32c115c12..1a3990ea1 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -214,8 +214,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
-        NixInt n;
-        if (string2Int(v->string.s, n)) return n;
+        if (auto n = string2Int<NixInt>(v->string.s))
+            return *n;
     }
     return def;
 }
@@ -228,8 +228,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
-        NixFloat n;
-        if (string2Float(v->string.s, n)) return n;
+        if (auto n = string2Float<NixFloat>(v->string.s))
+            return *n;
     }
     return def;
 }
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index bcb904c0d..d1003de57 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme
             if (name == "rev" || name == "narHash")
                 input.attrs.insert_or_assign(name, value);
             else if (name == "revCount" || name == "lastModified") {
-                uint64_t n;
-                if (!string2Int(value, n))
+                if (auto n = string2Int<uint64_t>(value))
+                    input.attrs.insert_or_assign(name, *n);
+                else
                     throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input.attrs.insert_or_assign(name, n);
             }
             else
                 throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 223020378..f1feeddd6 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -219,10 +219,10 @@ LegacyArgs::LegacyArgs(const std::string & programName,
             .description = description,
             .labels = {"n"},
             .handler = {[=](std::string s) {
-                unsigned int n;
-                if (!string2Int(s, n))
+                if (auto n = string2Int<unsigned int>(s))
+                    settings.set(dest, std::to_string(*n));
+                else
                     throw UsageError("'%s' is not an integer", s);
-                settings.set(dest, std::to_string(n));
             }}
         });
     };
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index ffae5d796..38f627b44 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -70,10 +70,9 @@ template<class N> N getIntArg(const string & opt,
             s.resize(s.size() - 1);
         }
     }
-    N n;
-    if (!string2Int(s, n))
-        throw UsageError("'%1%' requires an integer argument", opt);
-    return n * multiplier;
+    if (auto n = string2Int<N>(s))
+        return *n * multiplier;
+    throw UsageError("'%1%' requires an integer argument", opt);
 }
 
 
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index af3ab87a9..35f365795 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1699,12 +1699,10 @@ void DerivationGoal::startBuilder()
             userNamespaceSync.writeSide = -1;
         });
 
-        pid_t tmp;
         auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
         assert(ss.size() == 2);
         usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
-        pid = tmp;
+        pid = string2Int<pid_t>(ss[1]).value();
 
         if (usingUserNamespace) {
             /* Set the UID/GID mapping of the builder's user namespace
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index ad66ef8a8..0531aad9f 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -228,8 +228,12 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
 void MaxBuildJobsSetting::set(const std::string & str, bool append)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
-    else if (!string2Int(str, value))
-        throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
+    else {
+        if (auto n = string2Int<decltype(value)>(str))
+            value = *n;
+        else
+            throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
+    }
 }
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 702e7b136..c61f34275 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -66,8 +66,10 @@ int getSchema(Path schemaPath)
     int curSchema = 0;
     if (pathExists(schemaPath)) {
         string s = readFile(schemaPath);
-        if (!string2Int(s, curSchema))
+        auto n = string2Int<int>(s);
+        if (!n)
             throw Error("'%1%' is corrupt", schemaPath);
+        curSchema = *n;
     }
     return curSchema;
 }
diff --git a/src/libstore/names.cc b/src/libstore/names.cc
index 41e28dc99..ce808accc 100644
--- a/src/libstore/names.cc
+++ b/src/libstore/names.cc
@@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p,
 
 static bool componentsLT(const string & c1, const string & c2)
 {
-    int n1, n2;
-    bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
+    auto n1 = string2Int<int>(c1);
+    auto n2 = string2Int<int>(c2);
 
-    if (c1Num && c2Num) return n1 < n2;
-    else if (c1 == "" && c2Num) return true;
+    if (n1 && n2) return *n1 < *n2;
+    else if (c1 == "" && n2) return true;
     else if (c1 == "pre" && c2 != "pre") return true;
     else if (c2 == "pre") return false;
     /* Assume that `2.3a' < `2.3.1'. */
-    else if (c2Num) return true;
-    else if (c1Num) return false;
+    else if (n2) return true;
+    else if (n1) return false;
     else return c1 < c2;
 }
 
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 3454f34bb..49079388a 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "FileHash")
             fileHash = parseHashField(value);
         else if (name == "FileSize") {
-            if (!string2Int(value, fileSize)) throw corrupt();
+            auto n = string2Int<decltype(fileSize)>(value);
+            if (!n) throw corrupt();
+            fileSize = *n;
         }
         else if (name == "NarHash") {
             narHash = parseHashField(value);
             haveNarHash = true;
         }
         else if (name == "NarSize") {
-            if (!string2Int(value, narSize)) throw corrupt();
+            auto n = string2Int<decltype(narSize)>(value);
+            if (!n) throw corrupt();
+            narSize = *n;
         }
         else if (name == "References") {
             auto refs = tokenizeString<Strings>(value, " ");
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index ed10dd519..5d1723886 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -21,9 +21,8 @@ static std::optional<GenerationNumber> parseName(const string & profileName, con
     string s = string(name, profileName.size() + 1);
     string::size_type p = s.find("-link");
     if (p == string::npos) return {};
-    unsigned int n;
-    if (string2Int(string(s, 0, p), n) && n >= 0)
-        return n;
+    if (auto n = string2Int<unsigned int>(s.substr(0, p)))
+        return *n;
     else
         return {};
 }
@@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
 {
     time_t curTime = time(0);
     string strDays = string(timeSpec, 0, timeSpec.size() - 1);
-    int days;
+    auto days = string2Int<int>(strDays);
 
-    if (!string2Int(strDays, days) || days < 1)
+    if (!days || *days < 1)
         throw Error("invalid number of days specifier '%1%'", timeSpec);
 
-    time_t oldTime = curTime - days * 24 * 3600;
+    time_t oldTime = curTime - *days * 24 * 3600;
 
     deleteGenerationsOlderThan(profile, oldTime, dryRun);
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7aca22bde..01e2fcc7b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -932,19 +932,20 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
         getline(str, s);
         auto narHash = Hash::parseAny(s, htSHA256);
         getline(str, s);
-        uint64_t narSize;
-        if (!string2Int(s, narSize)) throw Error("number expected");
-        hashGiven = { narHash, narSize };
+        auto narSize = string2Int<uint64_t>(s);
+        if (!narSize) throw Error("number expected");
+        hashGiven = { narHash, *narSize };
     }
     ValidPathInfo info(store.parseStorePath(path), hashGiven->first);
     info.narSize = hashGiven->second;
     std::string deriver;
     getline(str, deriver);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    string s; int n;
+    string s;
     getline(str, s);
-    if (!string2Int(s, n)) throw Error("number expected");
-    while (n--) {
+    auto n = string2Int<int>(s);
+    if (!n) throw Error("number expected");
+    while ((*n)--) {
         getline(str, s);
         info.references.insert(store.parseStorePath(s));
     }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 62b9516d8..823d843aa 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -87,7 +87,9 @@ protected:
         template<class I>
         Handler(I * dest)
             : fun([=](std::vector<std::string> ss) {
-                if (!string2Int(ss[0], *dest))
+                if (auto n = string2Int<I>(ss[0]))
+                    *dest = *n;
+                else
                     throw UsageError("'%s' is not an integer", ss[0]);
               })
             , arity(1)
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 7af3e7883..7467e5ac0 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -230,7 +230,9 @@ template<typename T>
 void BaseSetting<T>::set(const std::string & str, bool append)
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
-    if (!string2Int(str, value))
+    if (auto n = string2Int<T>(str))
+        value = *n;
+    else
         throw UsageError("setting '%s' has invalid value '%s'", name, str);
 }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 0f82bed78..7a4d5fe92 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -397,21 +397,27 @@ bool statusOk(int status);
 
 
 /* Parse a string into an integer. */
-template<class N> bool string2Int(const string & s, N & n)
+template<class N>
+std::optional<N> string2Int(const std::string & s)
 {
-    if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed)
-        return false;
+    if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
+        return {};
     std::istringstream str(s);
+    N n;
     str >> n;
-    return str && str.get() == EOF;
+    if (str && str.get() == EOF) return n;
+    return {};
 }
 
 /* Parse a string into a float. */
-template<class N> bool string2Float(const string & s, N & n)
+template<class N>
+std::optional<N> string2Float(const string & s)
 {
     std::istringstream str(s);
+    N n;
     str >> n;
-    return str && str.get() == EOF;
+    if (str && str.get() == EOF) return n;
+    return {};
 }
 
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 6c2e075ed..9963f05d9 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg
     if (opArgs.size() != 1)
         throw UsageError("exactly one argument expected");
 
-    GenerationNumber dstGen;
-    if (!string2Int(opArgs.front(), dstGen))
+    if (auto dstGen = string2Int<GenerationNumber>(opArgs.front()))
+        switchGeneration(globals, *dstGen);
+    else
         throw UsageError("expected a generation number");
-
-    switchGeneration(globals, dstGen);
 }
 
 
@@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
         if(opArgs.front().size() < 2)
             throw Error("invalid number of generations ‘%1%’", opArgs.front());
         string str_max = string(opArgs.front(), 1, opArgs.front().size());
-        GenerationNumber max;
-        if (!string2Int(str_max, max) || max == 0)
+        auto max = string2Int<GenerationNumber>(str_max);
+        if (!max || *max == 0)
             throw Error("invalid number of generations to keep ‘%1%’", opArgs.front());
-        deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
+        deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun);
     } else {
         std::set<GenerationNumber> gens;
         for (auto & i : opArgs) {
-            GenerationNumber n;
-            if (!string2Int(i, n))
+            if (auto n = string2Int<GenerationNumber>(i))
+                gens.insert(*n);
+            else
                 throw UsageError("invalid generation number '%1%'", i);
-            gens.insert(n);
         }
         deleteGenerations(globals.profile, gens, globals.dryRun);
     }
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index d8d2b3a70..8cdd34a20 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -209,9 +209,8 @@ public:
         std::vector<Matcher> res;
 
         for (auto & s : _matchers) {
-            size_t n;
-            if (string2Int(s, n))
-                res.push_back(n);
+            if (auto n = string2Int<size_t>(s))
+                res.push_back(*n);
             else if (store->isStorePath(s))
                 res.push_back(s);
             else

From 17beae299d5e6bb511c453d0b9d0d7ef906b3d14 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Jan 2021 12:51:19 +0100
Subject: [PATCH 653/882] Support binary unit prefixes in command line
 arguments

---
 src/libmain/shared.cc |  6 ++----
 src/libmain/shared.hh | 17 +----------------
 src/libutil/args.hh   |  5 +----
 src/libutil/util.hh   | 28 +++++++++++++++++++++++++---
 4 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index f1feeddd6..e797c2fb9 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -219,10 +219,8 @@ LegacyArgs::LegacyArgs(const std::string & programName,
             .description = description,
             .labels = {"n"},
             .handler = {[=](std::string s) {
-                if (auto n = string2Int<unsigned int>(s))
-                    settings.set(dest, std::to_string(*n));
-                else
-                    throw UsageError("'%s' is not an integer", s);
+                auto n = string2IntWithUnitPrefix<uint64_t>(s);
+                settings.set(dest, std::to_string(n));
             }}
         });
     };
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 38f627b44..edc7b5efa 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -57,22 +57,7 @@ template<class N> N getIntArg(const string & opt,
 {
     ++i;
     if (i == end) throw UsageError("'%1%' requires an argument", opt);
-    string s = *i;
-    N multiplier = 1;
-    if (allowUnit && !s.empty()) {
-        char u = std::toupper(*s.rbegin());
-        if (std::isalpha(u)) {
-            if (u == 'K') multiplier = 1ULL << 10;
-            else if (u == 'M') multiplier = 1ULL << 20;
-            else if (u == 'G') multiplier = 1ULL << 30;
-            else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError("invalid unit specifier '%1%'", u);
-            s.resize(s.size() - 1);
-        }
-    }
-    if (auto n = string2Int<N>(s))
-        return *n * multiplier;
-    throw UsageError("'%1%' requires an integer argument", opt);
+    return string2IntWithUnitPrefix<N>(*i);
 }
 
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 823d843aa..3783bc84f 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -87,10 +87,7 @@ protected:
         template<class I>
         Handler(I * dest)
             : fun([=](std::vector<std::string> ss) {
-                if (auto n = string2Int<I>(ss[0]))
-                    *dest = *n;
-                else
-                    throw UsageError("'%s' is not an integer", ss[0]);
+                *dest = string2IntWithUnitPrefix<I>(ss[0]);
               })
             , arity(1)
         { }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 7a4d5fe92..ab0bd865a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -401,12 +401,34 @@ template<class N>
 std::optional<N> string2Int(const std::string & s)
 {
     if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
-        return {};
+        return std::nullopt;
     std::istringstream str(s);
     N n;
     str >> n;
     if (str && str.get() == EOF) return n;
-    return {};
+    return std::nullopt;
+}
+
+/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
+   'T' denoting a binary unit prefix. */
+template<class N>
+N string2IntWithUnitPrefix(std::string s)
+{
+    N multiplier = 1;
+    if (!s.empty()) {
+        char u = std::toupper(*s.rbegin());
+        if (std::isalpha(u)) {
+            if (u == 'K') multiplier = 1ULL << 10;
+            else if (u == 'M') multiplier = 1ULL << 20;
+            else if (u == 'G') multiplier = 1ULL << 30;
+            else if (u == 'T') multiplier = 1ULL << 40;
+            else throw UsageError("invalid unit specifier '%1%'", u);
+            s.resize(s.size() - 1);
+        }
+    }
+    if (auto n = string2Int<N>(s))
+        return *n * multiplier;
+    throw UsageError("'%s' is not an integer", s);
 }
 
 /* Parse a string into a float. */
@@ -417,7 +439,7 @@ std::optional<N> string2Float(const string & s)
     N n;
     str >> n;
     if (str && str.get() == EOF) return n;
-    return {};
+    return std::nullopt;
 }
 
 

From e21aee58f6dd7785df50d5d2a473feb5f6b2ed4f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Jan 2021 14:17:06 +0100
Subject: [PATCH 654/882] Fix tests

---
 src/libutil/tests/tests.cc | 26 ++++++++------------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 35a5d27bb..58df9c5ac 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -320,20 +320,15 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(string2Float, emptyString) {
-        double n;
-        ASSERT_EQ(string2Float("", n), false);
+        ASSERT_EQ(string2Float<double>(""), std::nullopt);
     }
 
     TEST(string2Float, trivialConversions) {
-        double n;
-        ASSERT_EQ(string2Float("1.0", n), true);
-        ASSERT_EQ(n, 1.0);
+        ASSERT_EQ(string2Float<double>("1.0"), 1.0);
 
-        ASSERT_EQ(string2Float("0.0", n), true);
-        ASSERT_EQ(n, 0.0);
+        ASSERT_EQ(string2Float<double>("0.0"), 0.0);
 
-        ASSERT_EQ(string2Float("-100.25", n), true);
-        ASSERT_EQ(n, (-100.25));
+        ASSERT_EQ(string2Float<double>("-100.25"), -100.25);
     }
 
     /* ----------------------------------------------------------------------------
@@ -341,20 +336,15 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(string2Int, emptyString) {
-        double n;
-        ASSERT_EQ(string2Int("", n), false);
+        ASSERT_EQ(string2Int<int>(""), std::nullopt);
     }
 
     TEST(string2Int, trivialConversions) {
-        double n;
-        ASSERT_EQ(string2Int("1", n), true);
-        ASSERT_EQ(n, 1);
+        ASSERT_EQ(string2Int<int>("1"), 1);
 
-        ASSERT_EQ(string2Int("0", n), true);
-        ASSERT_EQ(n, 0);
+        ASSERT_EQ(string2Int<int>("0"), 0);
 
-        ASSERT_EQ(string2Int("-100", n), true);
-        ASSERT_EQ(n, (-100));
+        ASSERT_EQ(string2Int<int>("-100"), -100);
     }
 
     /* ----------------------------------------------------------------------------

From 1db3f84baccc30ac38227c1f7edc3bfbc8e5ff5b Mon Sep 17 00:00:00 2001
From: Danila <danila.fedorin@gmail.com>
Date: Fri, 8 Jan 2021 16:12:21 -0800
Subject: [PATCH 655/882] Upcase "Boolean" in Flake attribute type error

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libexpr/flake/flake.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 9f1e4063f..61aeae543 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(attr.name, attr.value->integer);
                         break;
                     default:
-                        throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected",
+                        throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
                             attr.name, showType(*attr.value));
                 }
             }

From fdcd62eec59485665b919c048874de05235b5971 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 10 Jan 2021 23:20:02 +0100
Subject: [PATCH 656/882] Add 'nix store gc' command

---
 src/nix/store-gc.cc | 43 +++++++++++++++++++++++++++++++++++++++++++
 src/nix/store-gc.md | 21 +++++++++++++++++++++
 tests/flakes.sh     |  6 +++---
 3 files changed, 67 insertions(+), 3 deletions(-)
 create mode 100644 src/nix/store-gc.cc
 create mode 100644 src/nix/store-gc.md

diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc
new file mode 100644
index 000000000..6e9607d03
--- /dev/null
+++ b/src/nix/store-gc.cc
@@ -0,0 +1,43 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdStoreGC : StoreCommand, MixDryRun
+{
+    GCOptions options;
+
+    CmdStoreGC()
+    {
+        addFlag({
+            .longName = "max",
+            .description = "stop after freeing `n` bytes of disk space",
+            .labels = {"n"},
+            .handler = {&options.maxFreed}
+        });
+    }
+
+    std::string description() override
+    {
+        return "perform garbage collection on a Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-gc.md"
+          ;
+    }
+
+    void run(ref<Store> store) override
+    {
+        options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead;
+        GCResults results;
+        PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
+        store->collectGarbage(options, results);
+    }
+};
+
+static auto rCmdStoreGC = registerCommand2<CmdStoreGC>({"store", "gc"});
diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md
new file mode 100644
index 000000000..956b3c872
--- /dev/null
+++ b/src/nix/store-gc.md
@@ -0,0 +1,21 @@
+R""(
+
+# Examples
+
+* Delete unreachable paths in the Nix store:
+
+  ```console
+  # nix store gc
+  ```
+
+* Delete up to 1 gigabyte of garbage:
+
+  ```console
+  # nix store gc --max 1G
+  ```
+
+# Description
+
+This command deletes unreachable paths in the Nix store.
+
+)""
diff --git a/tests/flakes.sh b/tests/flakes.sh
index 5aec563ac..2b7bcdd68 100644
--- a/tests/flakes.sh
+++ b/tests/flakes.sh
@@ -276,18 +276,18 @@ git -C $flake3Dir commit -m 'Add lockfile'
 # Test whether registry caching works.
 nix registry list --flake-registry file://$registry | grep -q flake3
 mv $registry $registry.tmp
-nix-store --gc
+nix store gc
 nix registry list --flake-registry file://$registry --refresh | grep -q flake3
 mv $registry.tmp $registry
 
 # Test whether flakes are registered as GC roots for offline use.
 # FIXME: use tarballs rather than git.
 rm -rf $TEST_HOME/.cache
-nix-store --gc # get rid of copies in the store to ensure they get fetched to our git cache
+nix store gc # get rid of copies in the store to ensure they get fetched to our git cache
 _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar
 mv $flake1Dir $flake1Dir.tmp
 mv $flake2Dir $flake2Dir.tmp
-nix-store --gc
+nix store gc
 _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar
 _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar --refresh
 mv $flake1Dir.tmp $flake1Dir

From 93ad6430edf3d7efa5948d1e0ca0447e4666b121 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Jan 2021 12:36:39 +0100
Subject: [PATCH 657/882] nix store prefetch-tarball -> nix flake prefetch

---
 src/nix/flake-prefetch.md         | 28 +++++++++++
 src/nix/flake.cc                  | 40 ++++++++++++++++
 src/nix/prefetch.cc               | 77 +++++++++----------------------
 src/nix/store-prefetch-tarball.md | 31 -------------
 4 files changed, 89 insertions(+), 87 deletions(-)
 create mode 100644 src/nix/flake-prefetch.md
 delete mode 100644 src/nix/store-prefetch-tarball.md

diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md
new file mode 100644
index 000000000..a1cf0289a
--- /dev/null
+++ b/src/nix/flake-prefetch.md
@@ -0,0 +1,28 @@
+R""(
+
+# Examples
+
+* Download a tarball and unpack it:
+
+  ```console
+  # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz
+  Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='
+  to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash
+  'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=').
+  ```
+
+* Download the `dwarffs` flake (looked up in the flake registry):
+
+  ```console
+  # nix flake prefetch dwarffs --json
+  {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0="
+  ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"}
+  ```
+
+# Description
+
+This command downloads the source tree denoted by flake reference
+*flake-url*. Note that this does not need to be a flake (i.e. it does
+not have to contain a `flake.nix` file).
+
+)""
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 2b91faa64..b73b9cf4e 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -960,6 +960,45 @@ struct CmdFlakeShow : FlakeCommand
     }
 };
 
+struct CmdFlakePrefetch : FlakeCommand, MixJSON
+{
+    CmdFlakePrefetch()
+    {
+    }
+
+    std::string description() override
+    {
+        return "download the source tree denoted by a flake reference into the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "flake-prefetch.md"
+          ;
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto originalRef = getFlakeRef();
+        auto resolvedRef = originalRef.resolve(store);
+        auto [tree, lockedRef] = resolvedRef.fetchTree(store);
+        auto hash = store->queryPathInfo(tree.storePath)->narHash;
+
+        if (json) {
+            auto res = nlohmann::json::object();
+            res["storePath"] = store->printStorePath(tree.storePath);
+            res["hash"] = hash.to_string(SRI, true);
+            logger->cout(res.dump());
+        } else {
+            notice("Downloaded '%s' to '%s' (hash '%s').",
+                lockedRef.to_string(),
+                store->printStorePath(tree.storePath),
+                hash.to_string(SRI, true));
+        }
+    }
+};
+
 struct CmdFlake : NixMultiCommand
 {
     CmdFlake()
@@ -973,6 +1012,7 @@ struct CmdFlake : NixMultiCommand
                 {"clone", []() { return make_ref<CmdFlakeClone>(); }},
                 {"archive", []() { return make_ref<CmdFlakeArchive>(); }},
                 {"show", []() { return make_ref<CmdFlakeShow>(); }},
+                {"prefetch", []() { return make_ref<CmdFlakePrefetch>(); }},
             })
     {
     }
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index 969299489..ce8c85ecf 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -246,17 +246,15 @@ static int main_nix_prefetch_url(int argc, char * * argv)
 
 static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);
 
-struct CmdStorePrefetch : StoreCommand, MixJSON
+struct CmdStorePrefetchFile : StoreCommand, MixJSON
 {
     std::string url;
     bool executable = false;
-    bool unpack;
     std::optional<std::string> name;
     HashType hashType = htSHA256;
     std::optional<Hash> expectedHash;
 
-    CmdStorePrefetch(bool unpack)
-        : unpack(unpack)
+    CmdStorePrefetchFile()
     {
         addFlag({
             .longName = "name",
@@ -267,7 +265,7 @@ struct CmdStorePrefetch : StoreCommand, MixJSON
 
         addFlag({
             .longName = "expected-hash",
-            .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file",
+            .description = "expected hash of the file",
             .labels = {"hash"},
             .handler = {[&](std::string s) {
                 expectedHash = Hash::parseAny(s, hashType);
@@ -276,14 +274,31 @@ struct CmdStorePrefetch : StoreCommand, MixJSON
 
         addFlag(Flag::mkHashTypeFlag("hash-type", &hashType));
 
+        addFlag({
+            .longName = "executable",
+            .description = "make the resulting file executable",
+            .handler = {&executable, true},
+        });
+
         expectArg("url", &url);
     }
 
     Category category() override { return catUtility; }
 
+    std::string description() override
+    {
+        return "download a file into the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-prefetch-file.md"
+          ;
+    }
     void run(ref<Store> store) override
     {
-        auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable);
+        auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable);
 
         if (json) {
             auto res = nlohmann::json::object();
@@ -299,54 +314,4 @@ struct CmdStorePrefetch : StoreCommand, MixJSON
     }
 };
 
-struct CmdStorePrefetchFile : CmdStorePrefetch
-{
-    CmdStorePrefetchFile()
-        : CmdStorePrefetch(false)
-    {
-        name = "source";
-
-        addFlag({
-            .longName = "executable",
-            .description = "make the resulting file executable",
-            .handler = {&executable, true},
-        });
-    }
-
-    std::string description() override
-    {
-        return "download a file into the Nix store";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "store-prefetch-file.md"
-          ;
-    }
-};
-
 static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"});
-
-struct CmdStorePrefetchTarball : CmdStorePrefetch
-{
-    CmdStorePrefetchTarball()
-        : CmdStorePrefetch(true)
-    {
-        name = "source";
-    }
-
-    std::string description() override
-    {
-        return "download and unpack a tarball into the Nix store";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "store-prefetch-tarball.md"
-          ;
-    }
-};
-
-static auto rCmdStorePrefetchTarball = registerCommand2<CmdStorePrefetchTarball>({"store", "prefetch-tarball"});
diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md
deleted file mode 100644
index 535d7e022..000000000
--- a/src/nix/store-prefetch-tarball.md
+++ /dev/null
@@ -1,31 +0,0 @@
-R""(
-
-# Examples
-
-* Download a tarball and unpack it:
-
-  ```console
-  # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz
-  Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz'
-  to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash
-  'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=').
-  ```
-
-* Download a tarball and unpack it, unless it already exists in the
-  Nix store:
-
-  ```console
-  # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \
-      --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=
-  ```
-
-# Description
-
-This command downloads a tarball or zip file from *url*, unpacks it,
-and adds the unpacked tree to the Nix store. It prints out the
-resulting store path and the NAR hash of that store path.
-
-The name component of the store path defaults to `source`, but this
-can be overriden using `--name`.
-
-)""

From 77c9ceda4be8dd304b07f654d8c059a01d839108 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Jan 2021 19:42:24 +0100
Subject: [PATCH 658/882] Tweak

---
 doc/manual/src/command-ref/nix-store.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 827adbd05..361c20cc9 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -226,7 +226,7 @@ control what gets deleted and in what order:
     or TiB units.
 
 The behaviour of the collector is also influenced by the
-`keep-outputs` and `keep-derivations` variables in the Nix
+`keep-outputs` and `keep-derivations` settings in the Nix
 configuration file.
 
 By default, the collector prints the total number of freed bytes when it

From 6254b1f5d298ff73127d7b0f0da48f142bdc753c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Jan 2021 19:46:17 +0100
Subject: [PATCH 659/882] Add 'nix store delete' command

---
 src/nix/store-delete.cc   | 45 +++++++++++++++++++++++++++++++++++++++
 src/nix/store-delete.md   | 24 +++++++++++++++++++++
 tests/multiple-outputs.sh |  2 +-
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100644 src/nix/store-delete.cc
 create mode 100644 src/nix/store-delete.md

diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc
new file mode 100644
index 000000000..f3677763c
--- /dev/null
+++ b/src/nix/store-delete.cc
@@ -0,0 +1,45 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdStoreDelete : StorePathsCommand
+{
+    GCOptions options { .action = GCOptions::gcDeleteSpecific };
+
+    CmdStoreDelete()
+    {
+        addFlag({
+            .longName = "ignore-liveness",
+            .description = "do not check whether the paths are reachable from a root",
+            .handler = {&options.ignoreLiveness, true}
+        });
+    }
+
+    std::string description() override
+    {
+        return "delete paths from the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-delete.md"
+          ;
+    }
+
+    void run(ref<Store> store, std::vector<StorePath> storePaths) override
+    {
+
+        for (auto & path : storePaths)
+            options.pathsToDelete.insert(path);
+
+        GCResults results;
+        PrintFreed freed(true, results);
+        store->collectGarbage(options, results);
+    }
+};
+
+static auto rCmdStoreDelete = registerCommand2<CmdStoreDelete>({"store", "delete"});
diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md
new file mode 100644
index 000000000..db535f87c
--- /dev/null
+++ b/src/nix/store-delete.md
@@ -0,0 +1,24 @@
+R""(
+
+# Examples
+
+* Delete a specific store path:
+
+  ```console
+  # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10
+  ```
+
+# Description
+
+This command deletes the store paths specified by *installables*. ,
+but only if it is safe to do so; that is, when the path is not
+reachable from a root of the garbage collector. This means that you
+can only delete paths that would also be deleted by `nix store
+gc`. Thus, `nix store delete` is a more targeted version of `nix store
+gc`.
+
+With the option `--ignore-liveness`, reachability from the roots is
+ignored. However, the path still won't be deleted if there are other
+paths in the store that refer to it (i.e., depend on it).
+
+)""
diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh
index 7a6ec181d..de573d4fa 100644
--- a/tests/multiple-outputs.sh
+++ b/tests/multiple-outputs.sh
@@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou
 
 # Delete one of the outputs and rebuild it.  This will cause a hash
 # rewrite.
-nix-store --delete $TEST_ROOT/result-second --ignore-liveness
+nix store delete $TEST_ROOT/result-second --ignore-liveness
 nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result
 [ "$(cat $TEST_ROOT/result-second/file)" = "second" ]
 [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ]

From 44fd7a05b655315fa0e6156ac33a1c5624460968 Mon Sep 17 00:00:00 2001
From: Rickard Nilsson <rickynils@gmail.com>
Date: Tue, 12 Jan 2021 01:28:00 +0100
Subject: [PATCH 660/882] Don't let 'preferLocalBuild' override 'max-jobs=0'

This resolves #3810 by changing the behavior of `max-jobs = 0`, so
that specifying the option also avoids local building of derivations
with the attribute `preferLocalBuild = true`.
---
 src/libstore/parsed-derivations.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index e7b7202d4..c5c3ae3dc 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const
         && !drv.isBuiltin())
         return false;
 
+    if (settings.maxBuildJobs.get() == 0
+        && !drv.isBuiltin())
+        return false;
+
     for (auto & feature : getRequiredSystemFeatures())
         if (!localStore.systemFeatures.get().count(feature)) return false;
 

From f69820417fa65dbfea88a5f4dd0ccb5376015a6b Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Mon, 11 Jan 2021 22:05:32 -0600
Subject: [PATCH 661/882] Set kern.curproc_arch_affinity=0 to escape Rosetta

By default, once you enter x86_64 Rosetta 2, macOS will try to run
everything in x86_64. So an x86_64 Nix will still try to use x86_64
even when system = aarch64-darwin. To avoid this we can set
kern.curproc_arch_affinity sysctl. With kern.curproc_arch_affinity=0,
we ignore this preference.

This is based on how
https://opensource.apple.com/source/system_cmds/system_cmds-880.40.5/arch.tproj/arch.c.auto.html
works. Completely undocumented, but seems to work!

Note, you can verify this works with this impure Nix expression:

```
  {
    a = derivation {
      name = "a";
      system = "aarch64-darwin";
      builder = "/bin/sh";
      args = [ "-e" (builtins.toFile "builder" ''
        [ "$(/usr/bin/arch)" = arm64 ]
        [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ]
        [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ]
        /usr/bin/touch $out
      '') ];
    };

    b = derivation {
      name = "b";
      system = "x86_64-darwin";
      builder = "/bin/sh";
      args = [ "-e" (builtins.toFile "builder" ''
        [ "$(/usr/bin/arch)" = i386 ]
        [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ]
        [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ]
        /usr/bin/touch $out
      '') ];
    };
  }
```
---
 src/libstore/build/derivation-goal.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 35f365795..a02ddb950 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -52,6 +52,7 @@
 
 #if __APPLE__
 #include <spawn.h>
+#include <sys/sysctl.h>
 #endif
 
 #include <pwd.h>
@@ -2869,6 +2870,10 @@ void DerivationGoal::runChild()
             throw SysError("failed to initialize builder");
 
         if (drv->platform == "aarch64-darwin") {
+            // Unset kern.curproc_arch_affinity so we can escape Rosetta
+            int affinity = 0;
+            sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
+
             cpu_type_t cpu = CPU_TYPE_ARM64;
             posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
         } else if (drv->platform == "x86_64-darwin") {

From 29007f8bc6ea42ae1f8311f00c0b5e14f04ec9e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jan 2021 19:57:05 +0100
Subject: [PATCH 662/882] nix profile info -> nix profile list

---
 src/nix/{profile-info.md => profile-list.md} | 2 +-
 src/nix/profile.cc                           | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename src/nix/{profile-info.md => profile-list.md} (98%)

diff --git a/src/nix/profile-info.md b/src/nix/profile-list.md
similarity index 98%
rename from src/nix/profile-info.md
rename to src/nix/profile-list.md
index a0c04fc8c..5c29c0b02 100644
--- a/src/nix/profile-info.md
+++ b/src/nix/profile-list.md
@@ -5,7 +5,7 @@ R""(
 * Show what packages are installed in the default profile:
 
   ```console
-  # nix profile info
+  # nix profile list
   0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1
   1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027
   2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 8cdd34a20..ac60d336c 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -336,7 +336,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
     }
 };
 
-struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
+struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
 {
     std::string description() override
     {
@@ -346,7 +346,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
     std::string doc() override
     {
         return
-          #include "profile-info.md"
+          #include "profile-list.md"
           ;
     }
 
@@ -408,7 +408,7 @@ struct CmdProfile : NixMultiCommand
               {"install", []() { return make_ref<CmdProfileInstall>(); }},
               {"remove", []() { return make_ref<CmdProfileRemove>(); }},
               {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }},
-              {"info", []() { return make_ref<CmdProfileInfo>(); }},
+              {"list", []() { return make_ref<CmdProfileList>(); }},
               {"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }},
           })
     { }

From 2f463e90ed077e066455a9ef6e024b18fd61c4de Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jan 2021 23:51:07 +0100
Subject: [PATCH 663/882] Add 'nix profile history' command

Replaces 'nix-env --list-generations'. Similar to 'nix profile
diff-closures' but shows only the changes in top-level packages.
---
 src/nix/command.hh         |   2 +
 src/nix/profile-history.md |  26 +++++++++
 src/nix/profile.cc         | 114 +++++++++++++++++++++++++++++++++++++
 3 files changed, 142 insertions(+)
 create mode 100644 src/nix/profile-history.md

diff --git a/src/nix/command.hh b/src/nix/command.hh
index 6882db195..3aae57edd 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -261,6 +261,8 @@ void completeFlakeRefWithFragment(
     const Strings & defaultFlakeAttrPaths,
     std::string_view prefix);
 
+std::string showVersions(const std::set<std::string> & versions);
+
 void printClosureDiff(
     ref<Store> store,
     const StorePath & beforePath,
diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md
new file mode 100644
index 000000000..d0fe40c82
--- /dev/null
+++ b/src/nix/profile-history.md
@@ -0,0 +1,26 @@
+R""(
+
+# Examples
+
+* Show the changes between each version of your default profile:
+
+  ```console
+  # nix profile history
+  Version 508 -> 509:
+    flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13
+
+  Version 509 -> 510:
+    flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211
+  ```
+
+# Description
+
+This command shows what packages were added, removed or upgraded
+between subsequent versions of a profile. It only shows top-level
+packages, not dependencies; for that, use [`nix profile
+diff-closures`](./nix3-profile-diff-closures.md).
+
+The addition of a package to a profile is denoted by the string `∅ ->`
+*version*, whereas the removal is denoted by *version* `-> ∅`.
+
+)""
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index ac60d336c..ca95817d0 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -8,6 +8,7 @@
 #include "flake/flakeref.hh"
 #include "../nix-env/user-env.hh"
 #include "profiles.hh"
+#include "names.hh"
 
 #include <nlohmann/json.hpp>
 #include <regex>
@@ -21,6 +22,13 @@ struct ProfileElementSource
     FlakeRef resolvedRef;
     std::string attrPath;
     // FIXME: output names
+
+    bool operator < (const ProfileElementSource & other) const
+    {
+        return
+            std::pair(originalRef.to_string(), attrPath) <
+            std::pair(other.originalRef.to_string(), other.attrPath);
+    }
 };
 
 struct ProfileElement
@@ -29,6 +37,29 @@ struct ProfileElement
     std::optional<ProfileElementSource> source;
     bool active = true;
     // FIXME: priority
+
+    std::string describe() const
+    {
+        if (source)
+            return fmt("%s#%s", source->originalRef, source->attrPath);
+        StringSet names;
+        for (auto & path : storePaths)
+            names.insert(DrvName(path.name()).name);
+        return concatStringsSep(", ", names);
+    }
+
+    std::string versions() const
+    {
+        StringSet versions;
+        for (auto & path : storePaths)
+            versions.insert(DrvName(path.name()).version);
+        return showVersions(versions);
+    }
+
+    bool operator < (const ProfileElement & other) const
+    {
+        return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
+    }
 };
 
 struct ProfileManifest
@@ -142,6 +173,46 @@ struct ProfileManifest
 
         return std::move(info.path);
     }
+
+    static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent)
+    {
+        auto prevElems = prev.elements;
+        std::sort(prevElems.begin(), prevElems.end());
+
+        auto curElems = cur.elements;
+        std::sort(curElems.begin(), curElems.end());
+
+        auto i = prevElems.begin();
+        auto j = curElems.begin();
+
+        bool changes = false;
+
+        while (i != prevElems.end() || j != curElems.end()) {
+            if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) {
+                std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions());
+                changes = true;
+                ++j;
+            }
+            else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) {
+                std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions());
+                changes = true;
+                ++i;
+            }
+            else {
+                auto v1 = i->versions();
+                auto v2 = j->versions();
+                if (v1 != v2) {
+                    std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2);
+                    changes = true;
+                }
+                ++i;
+                ++j;
+            }
+        }
+
+        if (!changes)
+            std::cout << fmt("%sNo changes.\n", indent);
+    }
 };
 
 struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
@@ -401,6 +472,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
     }
 };
 
+struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile
+{
+    std::string description() override
+    {
+        return "show all versions of a profile";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "profile-history.md"
+          ;
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto [gens, curGen] = findGenerations(*profile);
+
+        std::optional<std::pair<Generation, ProfileManifest>> prevGen;
+        bool first = true;
+
+        for (auto & gen : gens) {
+            ProfileManifest manifest(*getEvalState(), gen.path);
+
+            if (!first) std::cout << "\n";
+            first = false;
+
+            if (prevGen)
+                std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number);
+            else
+                std::cout << fmt("Version %d:\n", gen.number);
+
+            ProfileManifest::printDiff(
+                prevGen ? prevGen->second : ProfileManifest(),
+                manifest,
+                "  ");
+
+            prevGen = {gen, std::move(manifest)};
+        }
+    }
+};
+
 struct CmdProfile : NixMultiCommand
 {
     CmdProfile()
@@ -410,6 +523,7 @@ struct CmdProfile : NixMultiCommand
               {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }},
               {"list", []() { return make_ref<CmdProfileList>(); }},
               {"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }},
+              {"history", []() { return make_ref<CmdProfileHistory>(); }},
           })
     { }
 

From 0ca1a5013269060919393afaa708640f574ab350 Mon Sep 17 00:00:00 2001
From: Rickard Nilsson <rickynils@gmail.com>
Date: Wed, 13 Jan 2021 10:13:51 +0100
Subject: [PATCH 664/882] Remove a redundant condition in
 DerivationGoal::tryLocalBuild()

---
 src/libstore/build/derivation-goal.cc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 35f365795..415a55d37 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -679,13 +679,9 @@ void DerivationGoal::tryToBuild()
 }
 
 void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
+    /* Make sure that we are allowed to start a build. */
     unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
+    if (curBuilds >= settings.maxBuildJobs) {
         worker.waitForBuildSlot(shared_from_this());
         outputLocks.unlock();
         return;

From 3da9a9241cb9f8c284426c220ea285398d0328dd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jan 2021 14:18:04 +0100
Subject: [PATCH 665/882] Convert option descriptions to Markdown

---
 src/libexpr/common-eval-args.cc | 10 +++++-----
 src/libmain/common-args.cc      | 13 ++++++-------
 src/libmain/common-args.hh      |  4 ++--
 src/libmain/shared.cc           | 20 ++++++++++----------
 src/nix/add-to-store.cc         |  2 +-
 src/nix/build.cc                |  6 +++---
 src/nix/bundle.cc               |  6 +++---
 src/nix/command.cc              | 14 +++++++-------
 src/nix/copy.cc                 |  8 ++++----
 src/nix/develop.cc              | 16 ++++++++--------
 src/nix/eval.cc                 |  6 +++---
 src/nix/flake.cc                |  8 ++++----
 src/nix/hash.cc                 | 19 +++++++++----------
 src/nix/installables.cc         | 26 +++++++++++++-------------
 src/nix/ls.cc                   |  6 +++---
 src/nix/main.cc                 | 14 +++++++-------
 src/nix/path-info.cc            |  8 ++++----
 src/nix/prefetch.cc             |  8 +++++---
 src/nix/run.cc                  |  2 +-
 src/nix/show-derivation.cc      |  2 +-
 src/nix/sigs.cc                 |  6 +++---
 src/nix/store-delete.cc         |  2 +-
 src/nix/store-gc.cc             |  2 +-
 src/nix/upgrade-nix.cc          |  4 ++--
 src/nix/verify.cc               |  8 ++++----
 src/nix/why-depends.cc          |  2 +-
 26 files changed, 111 insertions(+), 111 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 10c1a6975..ffe782454 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs()
 {
     addFlag({
         .longName = "arg",
-        .description = "argument to be passed to Nix functions",
+        .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .labels = {"name", "expr"},
         .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
     });
 
     addFlag({
         .longName = "argstr",
-        .description = "string-valued argument to be passed to Nix functions",
+        .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .labels = {"name", "string"},
         .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
     });
@@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "include",
         .shortName = 'I',
-        .description = "add a path to the list of locations used to look up `<...>` file names",
+        .description = "Add *path* to the list of locations used to look up `<...>` file names.",
         .labels = {"path"},
         .handler = {[&](std::string s) { searchPath.push_back(s); }}
     });
 
     addFlag({
         .longName = "impure",
-        .description = "allow access to mutable paths and repositories",
+        .description = "Allow access to mutable paths and repositories.",
         .handler = {[&]() {
             evalSettings.pureEval = false;
         }},
@@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs()
 
     addFlag({
         .longName = "override-flake",
-        .description = "override a flake registry value",
+        .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.",
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
             auto from = parseFlakeRef(_from, absPath("."));
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 3e4e475e5..bd5573e5d 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     addFlag({
         .longName = "verbose",
         .shortName = 'v',
-        .description = "increase verbosity level",
+        .description = "Increase the logging verbosity level.",
         .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
     });
 
     addFlag({
         .longName = "quiet",
-        .description = "decrease verbosity level",
+        .description = "Decrease the logging verbosity level.",
         .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
     });
 
     addFlag({
         .longName = "debug",
-        .description = "enable debug output",
+        .description = "Set the logging verbosity level to 'debug'.",
         .handler = {[]() { verbosity = lvlDebug; }},
     });
 
     addFlag({
         .longName = "option",
-        .description = "set a Nix configuration option (overriding `nix.conf`)",
+        .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
         .labels = {"name", "value"},
         .handler = {[](std::string name, std::string value) {
             try {
@@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "log-format",
-        .description = "format of log output; `raw`, `internal-json`, `bar` "
-                        "or `bar-with-logs`",
+        .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.",
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });
@@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     addFlag({
         .longName = "max-jobs",
         .shortName = 'j',
-        .description = "maximum number of parallel builds",
+        .description = "The maximum number of parallel builds.",
         .labels = Strings{"jobs"},
         .handler = {[=](std::string s) {
             settings.set("max-jobs", s);
diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh
index a4de3dccf..47f341619 100644
--- a/src/libmain/common-args.hh
+++ b/src/libmain/common-args.hh
@@ -16,7 +16,7 @@ struct MixDryRun : virtual Args
 
     MixDryRun()
     {
-        mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun);
+        mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun);
     }
 };
 
@@ -26,7 +26,7 @@ struct MixJSON : virtual Args
 
     MixJSON()
     {
-        mkFlag(0, "json", "produce JSON output", &json);
+        mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json);
     }
 };
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index e797c2fb9..7e27e95c2 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -186,27 +186,27 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     addFlag({
         .longName = "no-build-output",
         .shortName = 'Q',
-        .description = "do not show build output",
+        .description = "Do not show build output.",
         .handler = {[&]() {setLogFormat(LogFormat::raw); }},
     });
 
     addFlag({
         .longName = "keep-failed",
         .shortName ='K',
-        .description = "keep temporary directories of failed builds",
+        .description = "Keep temporary directories of failed builds.",
         .handler = {&(bool&) settings.keepFailed, true},
     });
 
     addFlag({
         .longName = "keep-going",
         .shortName ='k',
-        .description = "keep going after a build fails",
+        .description = "Keep going after a build fails.",
         .handler = {&(bool&) settings.keepGoing, true},
     });
 
     addFlag({
         .longName = "fallback",
-        .description = "build from source if substitution fails",
+        .description = "Build from source if substitution fails.",
         .handler = {&(bool&) settings.tryFallback, true},
     });
 
@@ -225,19 +225,19 @@ LegacyArgs::LegacyArgs(const std::string & programName,
         });
     };
 
-    intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
-    intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
-    intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
+    intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores");
+    intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time");
+    intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout");
 
-    mkFlag(0, "readonly-mode", "do not write to the Nix store",
+    mkFlag(0, "readonly-mode", "Do not write to the Nix store.",
         &settings.readOnlyMode);
 
-    mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
+    mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.",
         &gcWarning, false);
 
     addFlag({
         .longName = "store",
-        .description = "URI of the Nix store to use",
+        .description = "The URL of the Nix store to use.",
         .labels = {"store-uri"},
         .handler = {&(std::string&) settings.storeUri},
     });
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index ea4bbbab9..2ae042789 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         addFlag({
             .longName = "name",
             .shortName = 'n',
-            .description = "name component of the store path",
+            .description = "Override the name component of the store path. It defaults to the base name of *path*.",
             .labels = {"name"},
             .handler = {&namePart},
         });
diff --git a/src/nix/build.cc b/src/nix/build.cc
index c2974d983..4cb8ade08 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
         addFlag({
             .longName = "out-link",
             .shortName = 'o',
-            .description = "path of the symlink to the build result",
+            .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.",
             .labels = {"path"},
             .handler = {&outLink},
             .completer = completePath
@@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         addFlag({
             .longName = "no-link",
-            .description = "do not create a symlink to the build result",
+            .description = "Do not create symlinks to the build results.",
             .handler = {&outLink, Path("")},
         });
 
         addFlag({
             .longName = "rebuild",
-            .description = "rebuild an already built package and compare the result to the existing store paths",
+            .description = "Rebuild an already built package and compare the result to the existing store paths.",
             .handler = {&buildMode, bmCheck},
         });
     }
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 5f558b01e..1789e4598 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand
     {
         addFlag({
             .longName = "bundler",
-            .description = "use custom bundler",
+            .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler),
             .labels = {"flake-url"},
             .handler = {&bundler},
             .completer = {[&](size_t, std::string_view prefix) {
@@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand
         addFlag({
             .longName = "out-link",
             .shortName = 'o',
-            .description = "path of the symlink to the build result",
+            .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.",
             .labels = {"path"},
             .handler = {&outLink},
             .completer = completePath
@@ -90,7 +90,7 @@ struct CmdBundle : InstallableCommand
         mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
 
         arg->attrs->sort();
- 
+
         auto vRes = evalState->allocValue();
         evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 596217775..ba58c7d6b 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive)
     if (recursive)
         addFlag({
             .longName = "no-recursive",
-            .description = "apply operation to specified paths only",
+            .description = "Apply operation to specified paths only.",
             .handler = {&this->recursive, false},
         });
     else
         addFlag({
             .longName = "recursive",
             .shortName = 'r',
-            .description = "apply operation to closure of the specified paths",
+            .description = "Apply operation to closure of the specified paths.",
             .handler = {&this->recursive, true},
         });
 
-    mkFlag(0, "all", "apply operation to the entire store", &all);
+    mkFlag(0, "all", "Apply the operation to every store path.", &all);
 }
 
 void StorePathsCommand::run(ref<Store> store)
@@ -133,7 +133,7 @@ MixProfile::MixProfile()
 {
     addFlag({
         .longName = "profile",
-        .description = "profile to update",
+        .description = "The profile to update.",
         .labels = {"path"},
         .handler = {&profile},
         .completer = completePath
@@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
     addFlag({
         .longName = "ignore-environment",
         .shortName = 'i',
-        .description = "clear the entire environment (except those specified with --keep)",
+        .description = "Clear the entire environment (except those specified with `--keep`).",
         .handler = {&ignoreEnvironment, true},
     });
 
     addFlag({
         .longName = "keep",
         .shortName = 'k',
-        .description = "keep specified environment variable",
+        .description = "Keep the environment variable *name*.",
         .labels = {"name"},
         .handler = {[&](std::string s) { keep.insert(s); }},
     });
@@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
     addFlag({
         .longName = "unset",
         .shortName = 'u',
-        .description = "unset specified environment variable",
+        .description = "Unset the environment variable *name*.",
         .labels = {"name"},
         .handler = {[&](std::string s) { unset.insert(s); }},
     });
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 2394eb46d..f15031a45 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand
     {
         addFlag({
             .longName = "from",
-            .description = "URI of the source Nix store",
+            .description = "URL of the source Nix store.",
             .labels = {"store-uri"},
             .handler = {&srcUri},
         });
 
         addFlag({
             .longName = "to",
-            .description = "URI of the destination Nix store",
+            .description = "URL of the destination Nix store.",
             .labels = {"store-uri"},
             .handler = {&dstUri},
         });
 
         addFlag({
             .longName = "no-check-sigs",
-            .description = "do not require that paths are signed by trusted keys",
+            .description = "Do not require that paths are signed by trusted keys.",
             .handler = {&checkSigs, NoCheckSigs},
         });
 
         addFlag({
             .longName = "substitute-on-destination",
             .shortName = 's',
-            .description = "whether to try substitutes on the destination store (only supported by SSH)",
+            .description = "Whether to try substitutes on the destination store (only supported by SSH stores).",
             .handler = {&substitute, Substitute},
         });
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index edd87f246..578258394 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -204,7 +204,7 @@ struct Common : InstallableCommand, MixProfile
     {
         addFlag({
             .longName = "redirect",
-            .description = "redirect a store path to a mutable location",
+            .description = "Redirect a store path to a mutable location.",
             .labels = {"installable", "outputs-dir"},
             .handler = {[&](std::string installable, std::string outputsDir) {
                 redirects.push_back({installable, outputsDir});
@@ -334,7 +334,7 @@ struct CmdDevelop : Common, MixEnvironment
         addFlag({
             .longName = "command",
             .shortName = 'c',
-            .description = "command and arguments to be executed instead of an interactive shell",
+            .description = "Instead of starting an interactive shell, start the specified command and arguments.",
             .labels = {"command", "args"},
             .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
@@ -344,38 +344,38 @@ struct CmdDevelop : Common, MixEnvironment
 
         addFlag({
             .longName = "phase",
-            .description = "phase to run (e.g. `build` or `configure`)",
+            .description = "The stdenv phase to run (e.g. `build` or `configure`).",
             .labels = {"phase-name"},
             .handler = {&phase},
         });
 
         addFlag({
             .longName = "configure",
-            .description = "run the configure phase",
+            .description = "Run the `configure` phase.",
             .handler = {&phase, {"configure"}},
         });
 
         addFlag({
             .longName = "build",
-            .description = "run the build phase",
+            .description = "Run the `build` phase.",
             .handler = {&phase, {"build"}},
         });
 
         addFlag({
             .longName = "check",
-            .description = "run the check phase",
+            .description = "Run the `check` phase.",
             .handler = {&phase, {"check"}},
         });
 
         addFlag({
             .longName = "install",
-            .description = "run the install phase",
+            .description = "Run the `install` phase.",
             .handler = {&phase, {"install"}},
         });
 
         addFlag({
             .longName = "installcheck",
-            .description = "run the installcheck phase",
+            .description = "Run the `installcheck` phase.",
             .handler = {&phase, {"installCheck"}},
         });
     }
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 321df7495..b5049ac65 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -18,18 +18,18 @@ struct CmdEval : MixJSON, InstallableCommand
 
     CmdEval()
     {
-        mkFlag(0, "raw", "print strings unquoted", &raw);
+        mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw);
 
         addFlag({
             .longName = "apply",
-            .description = "apply a function to each argument",
+            .description = "Apply the function *expr* to each argument.",
             .labels = {"expr"},
             .handler = {&apply},
         });
 
         addFlag({
             .longName = "write-to",
-            .description = "write a string or attrset of strings to 'path'",
+            .description = "Write a string or attrset of strings to *path*.",
             .labels = {"path"},
             .handler = {&writeTo},
         });
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index b73b9cf4e..4cd7d77a0 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -222,7 +222,7 @@ struct CmdFlakeCheck : FlakeCommand
     {
         addFlag({
             .longName = "no-build",
-            .description = "do not build checks",
+            .description = "Do not build checks.",
             .handler = {&build, false}
         });
     }
@@ -573,7 +573,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
         addFlag({
             .longName = "template",
             .shortName = 't',
-            .description = "the template to use",
+            .description = "The template to use.",
             .labels = {"template"},
             .handler = {&templateUrl},
             .completer = {[&](size_t, std::string_view prefix) {
@@ -717,7 +717,7 @@ struct CmdFlakeClone : FlakeCommand
         addFlag({
             .longName = "dest",
             .shortName = 'f',
-            .description = "destination path",
+            .description = "Clone the flake to path *dest*.",
             .labels = {"path"},
             .handler = {&destDir}
         });
@@ -807,7 +807,7 @@ struct CmdFlakeShow : FlakeCommand
     {
         addFlag({
             .longName = "legacy",
-            .description = "show the contents of the 'legacyPackages' output",
+            .description = "Show the contents of the `legacyPackages` output.",
             .handler = {&showLegacy, true}
         });
     }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 6fd791f41..79d506ace 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -19,15 +19,15 @@ struct CmdHashBase : Command
 
     CmdHashBase(FileIngestionMethod mode) : mode(mode)
     {
-        mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
-        mkFlag(0, "base64", "print hash in base-64", &base, Base64);
-        mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
-        mkFlag(0, "base16", "print hash in base-16", &base, Base16);
+        mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI);
+        mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64);
+        mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32);
+        mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16);
         addFlag(Flag::mkHashTypeFlag("type", &ht));
         #if 0
         mkFlag()
             .longName("modulo")
-            .description("compute hash modulo specified string")
+            .description("Compute the hash modulo specified the string.")
             .labels({"modulus"})
             .dest(&modulus);
         #endif
@@ -40,15 +40,14 @@ struct CmdHashBase : Command
 
     std::string description() override
     {
-        const char* d;
         switch (mode) {
         case FileIngestionMethod::Flat:
-            d = "print cryptographic hash of a regular file";
-            break;
+            return  "print cryptographic hash of a regular file";
         case FileIngestionMethod::Recursive:
-            d = "print cryptographic hash of the NAR serialisation of a path";
+            return "print cryptographic hash of the NAR serialisation of a path";
+        default:
+            assert(false);
         };
-        return d;
     }
 
     void run() override
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 3506c3fcc..50e3b29c4 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -60,37 +60,37 @@ MixFlakeOptions::MixFlakeOptions()
 {
     addFlag({
         .longName = "recreate-lock-file",
-        .description = "recreate lock file from scratch",
+        .description = "Recreate the flake's lock file from scratch.",
         .handler = {&lockFlags.recreateLockFile, true}
     });
 
     addFlag({
         .longName = "no-update-lock-file",
-        .description = "do not allow any updates to the lock file",
+        .description = "Do not allow any updates to the flake's lock file.",
         .handler = {&lockFlags.updateLockFile, false}
     });
 
     addFlag({
         .longName = "no-write-lock-file",
-        .description = "do not write the newly generated lock file",
+        .description = "Do not write the flake's newly generated lock file.",
         .handler = {&lockFlags.writeLockFile, false}
     });
 
     addFlag({
         .longName = "no-registries",
-        .description = "don't use flake registries",
+        .description = "Don't allow lookups in the flake registries.",
         .handler = {&lockFlags.useRegistries, false}
     });
 
     addFlag({
         .longName = "commit-lock-file",
-        .description = "commit changes to the lock file",
+        .description = "Commit changes to the flake's lock file.",
         .handler = {&lockFlags.commitLockFile, true}
     });
 
     addFlag({
         .longName = "update-input",
-        .description = "update a specific flake input",
+        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
         .labels = {"input-path"},
         .handler = {[&](std::string s) {
             lockFlags.inputUpdates.insert(flake::parseInputPath(s));
@@ -103,7 +103,7 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)",
+        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).",
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
             lockFlags.inputOverrides.insert_or_assign(
@@ -114,7 +114,7 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "inputs-from",
-        .description = "use the inputs of the specified flake as registry entries",
+        .description = "Use the inputs of the specified flake as registry entries.",
         .labels = {"flake-url"},
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
@@ -143,22 +143,22 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "file",
         .shortName = 'f',
-        .description = "evaluate *file* rather than the default",
+        .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
         .labels = {"file"},
         .handler = {&file},
         .completer = completePath
     });
 
     addFlag({
-        .longName ="expr",
-        .description = "evaluate attributes from *expr*",
+        .longName = "expr",
+        .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.",
         .labels = {"expr"},
         .handler = {&expr}
     });
 
     addFlag({
-        .longName ="derivation",
-        .description = "operate on the store derivation rather than its outputs",
+        .longName = "derivation",
+        .description = "Operate on the store derivation rather than its outputs.",
         .handler = {&operateOn, OperateOn::Derivation},
     });
 }
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index d48287f27..c0b1ecb32 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -17,9 +17,9 @@ struct MixLs : virtual Args, MixJSON
 
     MixLs()
     {
-        mkFlag('R', "recursive", "list subdirectories recursively", &recursive);
-        mkFlag('l', "long", "show more file information", &verbose);
-        mkFlag('d', "directory", "show directories rather than their contents", &showDirectory);
+        mkFlag('R', "recursive", "List subdirectories recursively.", &recursive);
+        mkFlag('l', "long", "Show detailed file information.", &verbose);
+        mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory);
     }
 
     void listText(ref<FSAccessor> accessor)
diff --git a/src/nix/main.cc b/src/nix/main.cc
index b2406fafe..803453dd5 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -69,15 +69,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
         addFlag({
             .longName = "help",
-            .description = "show usage information",
+            .description = "Show usage information.",
             .handler = {[&]() { if (!completions) showHelpAndExit(); }},
         });
 
         addFlag({
             .longName = "help-config",
-            .description = "show configuration options",
+            .description = "Show configuration settings.",
             .handler = {[&]() {
-                std::cout << "The following configuration options are available:\n\n";
+                std::cout << "The following configuration settings are available:\n\n";
                 Table2 tbl;
                 std::map<std::string, Config::SettingInfo> settings;
                 globalConfig.getSettings(settings);
@@ -91,25 +91,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         addFlag({
             .longName = "print-build-logs",
             .shortName = 'L',
-            .description = "print full build logs on stderr",
+            .description = "Print full build logs on standard error.",
             .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }},
         });
 
         addFlag({
             .longName = "version",
-            .description = "show version information",
+            .description = "Show version information.",
             .handler = {[&]() { if (!completions) printVersion(programName); }},
         });
 
         addFlag({
             .longName = "no-net",
-            .description = "disable substituters and consider all previously downloaded files up-to-date",
+            .description = "Disable substituters and consider all previously downloaded files up-to-date.",
             .handler = {[&]() { useNet = false; }},
         });
 
         addFlag({
             .longName = "refresh",
-            .description = "consider all previously downloaded files out-of-date",
+            .description = "Consider all previously downloaded files out-of-date.",
             .handler = {[&]() { refresh = true; }},
         });
     }
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 30b6a50f8..0fa88f1bf 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -18,10 +18,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
 
     CmdPathInfo()
     {
-        mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
-        mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
-        mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable);
-        mkFlag(0, "sigs", "show signatures", &showSigs);
+        mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize);
+        mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize);
+        mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable);
+        mkFlag(0, "sigs", "Show signatures.", &showSigs);
     }
 
     std::string description() override
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index ce8c85ecf..a831dcd15 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -258,14 +258,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
     {
         addFlag({
             .longName = "name",
-            .description = "store path name",
+            .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.",
             .labels = {"name"},
             .handler = {&name}
         });
 
         addFlag({
             .longName = "expected-hash",
-            .description = "expected hash of the file",
+            .description = "The expected hash of the file.",
             .labels = {"hash"},
             .handler = {[&](std::string s) {
                 expectedHash = Hash::parseAny(s, hashType);
@@ -276,7 +276,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
 
         addFlag({
             .longName = "executable",
-            .description = "make the resulting file executable",
+            .description =
+                "Make the resulting file executable. Note that this causes the "
+                "resulting hash to be a NAR hash rather than a flat file hash.",
             .handler = {&executable, true},
         });
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 1340dd46f..ec9388234 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -72,7 +72,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
         addFlag({
             .longName = "command",
             .shortName = 'c',
-            .description = "command and arguments to be executed; defaults to '$SHELL'",
+            .description = "Command and arguments to be executed, defaulting to `$SHELL`",
             .labels = {"command", "args"},
             .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 13f2c8e69..2588a011d 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -19,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand
         addFlag({
             .longName = "recursive",
             .shortName = 'r',
-            .description = "include the dependencies of the specified derivations",
+            .description = "Include the dependencies of the specified derivations.",
             .handler = {&recursive, true}
         });
     }
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 14e2c9761..4b6ead6c7 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand
         addFlag({
             .longName = "substituter",
             .shortName = 's',
-            .description = "use signatures from specified store",
+            .description = "Use signatures from specified store.",
             .labels = {"store-uri"},
             .handler = {[&](std::string s) { substituterUris.push_back(s); }},
         });
@@ -101,7 +101,7 @@ struct CmdSignPaths : StorePathsCommand
         addFlag({
             .longName = "key-file",
             .shortName = 'k',
-            .description = "file containing the secret signing key",
+            .description = "File containing the secret signing key.",
             .labels = {"file"},
             .handler = {&secretKeyFile},
             .completer = completePath
@@ -150,7 +150,7 @@ struct CmdKeyGenerateSecret : Command
     {
         addFlag({
             .longName = "key-name",
-            .description = "identifier of the key (e.g. `cache.example.org-1`)",
+            .description = "Identifier of the key (e.g. `cache.example.org-1`).",
             .labels = {"name"},
             .handler = {&keyName},
         });
diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc
index f3677763c..9c8fef191 100644
--- a/src/nix/store-delete.cc
+++ b/src/nix/store-delete.cc
@@ -13,7 +13,7 @@ struct CmdStoreDelete : StorePathsCommand
     {
         addFlag({
             .longName = "ignore-liveness",
-            .description = "do not check whether the paths are reachable from a root",
+            .description = "Do not check whether the paths are reachable from a root.",
             .handler = {&options.ignoreLiveness, true}
         });
     }
diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc
index 6e9607d03..a2d74066e 100644
--- a/src/nix/store-gc.cc
+++ b/src/nix/store-gc.cc
@@ -13,7 +13,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun
     {
         addFlag({
             .longName = "max",
-            .description = "stop after freeing `n` bytes of disk space",
+            .description = "Stop after freeing *n* bytes of disk space.",
             .labels = {"n"},
             .handler = {&options.maxFreed}
         });
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 79be31e73..299ea40aa 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         addFlag({
             .longName = "profile",
             .shortName = 'p',
-            .description = "the Nix profile to upgrade",
+            .description = "The path to the Nix profile to upgrade.",
             .labels = {"profile-dir"},
             .handler = {&profileDir}
         });
 
         addFlag({
             .longName = "nix-store-paths-url",
-            .description = "URL of the file that contains the store paths of the latest Nix release",
+            .description = "The URL of the file that contains the store paths of the latest Nix release.",
             .labels = {"url"},
             .handler = {&storePathsUrl}
         });
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 620109aac..b2963cf74 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -18,13 +18,13 @@ struct CmdVerify : StorePathsCommand
 
     CmdVerify()
     {
-        mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
-        mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
+        mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents);
+        mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust);
 
         addFlag({
             .longName = "substituter",
             .shortName = 's',
-            .description = "use signatures from specified store",
+            .description = "Use signatures from the specified store.",
             .labels = {"store-uri"},
             .handler = {[&](std::string s) { substituterUris.push_back(s); }}
         });
@@ -32,7 +32,7 @@ struct CmdVerify : StorePathsCommand
         addFlag({
             .longName = "sigs-needed",
             .shortName = 'n',
-            .description = "require that each path has at least N valid signatures",
+            .description = "Require that each path has at least *n* valid signatures.",
             .labels = {"n"},
             .handler = {&sigsNeeded}
         });
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 297b638cc..7a4ca5172 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand
         addFlag({
             .longName = "all",
             .shortName = 'a',
-            .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path",
+            .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.",
             .handler = {&all, true},
         });
     }

From 61216d32e1c0973424d549c9f3065426b51015c9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jan 2021 23:27:39 +0100
Subject: [PATCH 666/882] Add 'nix store repair' command

---
 src/libstore/local-store.hh |  4 +---
 src/libstore/store-api.hh   |  5 +++++
 src/nix-store/nix-store.cc  |  2 +-
 src/nix/store-delete.cc     |  1 -
 src/nix/store-repair.cc     | 27 +++++++++++++++++++++++++++
 src/nix/store-repair.md     | 32 ++++++++++++++++++++++++++++++++
 6 files changed, 66 insertions(+), 5 deletions(-)
 create mode 100644 src/nix/store-repair.cc
 create mode 100644 src/nix/store-repair.md

diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 6d29c5960..6c7ebac1e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -198,9 +198,7 @@ public:
 
     void vacuumDB();
 
-    /* Repair the contents of the given path by redownloading it using
-       a substituter (if available). */
-    void repairPath(const StorePath & path);
+    void repairPath(const StorePath & path) override;
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9bcff08eb..d1b83933a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -604,6 +604,11 @@ public:
     virtual ref<FSAccessor> getFSAccessor()
     { unsupported("getFSAccessor"); }
 
+    /* Repair the contents of the given path by redownloading it using
+       a substituter (if available). */
+    virtual void repairPath(const StorePath & path)
+    { unsupported("repairPath"); }
+
     /* Add signatures to the specified store path. The signatures are
        not verified. */
     virtual void addSignatures(const StorePath & storePath, const StringSet & sigs)
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e43788bc3..b97f684a4 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -757,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs)
         throw UsageError("no flags expected");
 
     for (auto & i : opArgs)
-        ensureLocalStore()->repairPath(store->followLinksToStorePath(i));
+        store->repairPath(store->followLinksToStorePath(i));
 }
 
 /* Optimise the disk space usage of the Nix store by hard-linking
diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc
index 9c8fef191..10245978e 100644
--- a/src/nix/store-delete.cc
+++ b/src/nix/store-delete.cc
@@ -32,7 +32,6 @@ struct CmdStoreDelete : StorePathsCommand
 
     void run(ref<Store> store, std::vector<StorePath> storePaths) override
     {
-
         for (auto & path : storePaths)
             options.pathsToDelete.insert(path);
 
diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc
new file mode 100644
index 000000000..1c7a4392e
--- /dev/null
+++ b/src/nix/store-repair.cc
@@ -0,0 +1,27 @@
+#include "command.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdStoreRepair : StorePathsCommand
+{
+    std::string description() override
+    {
+        return "repair store paths";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-repair.md"
+          ;
+    }
+
+    void run(ref<Store> store, std::vector<StorePath> storePaths) override
+    {
+        for (auto & path : storePaths)
+            store->repairPath(path);
+    }
+};
+
+static auto rStoreRepair = registerCommand2<CmdStoreRepair>({"store", "repair"});
diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md
new file mode 100644
index 000000000..92d2205a9
--- /dev/null
+++ b/src/nix/store-repair.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Repair a store path, after determining that it is corrupt:
+
+  ```console
+  # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10
+  path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was
+  modified! expected hash
+  'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got
+  'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl'
+
+  # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10
+  ```
+
+# Description
+
+This command attempts to "repair" the store paths specified by
+*installables* by redownloading them using the available
+substituters. If no substitutes are available, then repair is not
+possible.
+
+> **Warning**
+>
+> During repair, there is a very small time window during which the old
+> path (if it exists) is moved out of the way and replaced with the new
+> path. If repair is interrupted in between, then the system may be left
+> in a broken state (e.g., if the path contains a critical system
+> component like the GNU C Library).
+
+)""

From d33eca8539d2e66759f7b52fa7b0db4a6a1ba673 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jan 2021 23:31:18 +0100
Subject: [PATCH 667/882] Rename 'nix store sign-paths' to 'nix store sign'

---
 doc/manual/src/advanced-topics/post-build-hook.md | 4 ++--
 src/nix/key-generate-secret.md                    | 2 +-
 src/nix/main.cc                                   | 2 +-
 src/nix/sigs.cc                                   | 6 +++---
 tests/signing.sh                                  | 8 ++++----
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md
index bbdabed41..fcb52d878 100644
--- a/doc/manual/src/advanced-topics/post-build-hook.md
+++ b/doc/manual/src/advanced-topics/post-build-hook.md
@@ -53,7 +53,7 @@ set -f # disable globbing
 export IFS=' '
 
 echo "Signing paths" $OUT_PATHS
-nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+nix store sign --key-file /etc/nix/key.private $OUT_PATHS
 echo "Uploading paths" $OUT_PATHS
 exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
 ```
@@ -63,7 +63,7 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
 > The `$OUT_PATHS` variable is a space-separated list of Nix store
 > paths. In this case, we expect and want the shell to perform word
 > splitting to make each output path its own argument to `nix
-> sign-paths`. Nix guarantees the paths will not contain any spaces,
+> store sign`. Nix guarantees the paths will not contain any spaces,
 > however a store path might contain glob characters. The `set -f`
 > disables globbing in the shell.
 
diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md
index 6ff1e1c9b..4938f637c 100644
--- a/src/nix/key-generate-secret.md
+++ b/src/nix/key-generate-secret.md
@@ -12,7 +12,7 @@ R""(
 
   ```console
   # nix build nixpkgs#hello
-  # nix store sign-paths --key-file ./secret-key --recursive ./result
+  # nix store sign --key-file ./secret-key --recursive ./result
   ```
 
   Finally, we can verify the store paths using the corresponding
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 803453dd5..398526020 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -129,7 +129,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         {"make-content-addressable", {"store", "make-content-addressable"}},
         {"optimise-store", {"store", "optimise"}},
         {"ping-store", {"store", "ping"}},
-        {"sign-paths", {"store", "sign-paths"}},
+        {"sign-paths", {"store", "sign"}},
         {"to-base16", {"hash", "to-base16"}},
         {"to-base32", {"hash", "to-base32"}},
         {"to-base64", {"hash", "to-base64"}},
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 4b6ead6c7..3445182f2 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -92,11 +92,11 @@ struct CmdCopySigs : StorePathsCommand
 
 static auto rCmdCopySigs = registerCommand2<CmdCopySigs>({"store", "copy-sigs"});
 
-struct CmdSignPaths : StorePathsCommand
+struct CmdSign : StorePathsCommand
 {
     Path secretKeyFile;
 
-    CmdSignPaths()
+    CmdSign()
     {
         addFlag({
             .longName = "key-file",
@@ -140,7 +140,7 @@ struct CmdSignPaths : StorePathsCommand
     }
 };
 
-static auto rCmdSignPaths = registerCommand2<CmdSignPaths>({"store", "sign-paths"});
+static auto rCmdSign = registerCommand2<CmdSign>({"store", "sign"});
 
 struct CmdKeyGenerateSecret : Command
 {
diff --git a/tests/signing.sh b/tests/signing.sh
index bd6280cc6..6aafbeb91 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -47,8 +47,8 @@ expect 2 nix store verify -r $outPath2 --sigs-needed 1
 
 expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
-# Test "nix store sign-paths".
-nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2
+# Test "nix store sign".
+nix store sign --key-file $TEST_ROOT/sk1 $outPath2
 
 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
@@ -63,7 +63,7 @@ nix store verify $outPathCA
 nix store verify $outPathCA --sigs-needed 1000
 
 # Check that signing a content-addressed path doesn't overflow validSigs
-nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA
+nix store sign --key-file $TEST_ROOT/sk1 $outPathCA
 nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1
 
 # Copy to a binary cache.
@@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2)
 (! [[ $info =~ 'cache2.example.org' ]])
 
 # Verify that adding a signature to a path in a binary cache works.
-nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
+nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
 info=$(nix path-info --store file://$cacheDir --json $outPath2)
 [[ $info =~ 'cache1.example.org' ]]
 [[ $info =~ 'cache2.example.org' ]]

From 28ef6ebf914792f9e543e9778248f06e716a859d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jan 2021 23:51:27 +0100
Subject: [PATCH 668/882] Typo

---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index fc6195cf0..9227369b8 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -318,7 +318,7 @@ static int main_nix_daemon(int argc, char * * argv)
                 FdSource from(STDIN_FILENO);
                 FdSink to(STDOUT_FILENO);
                 /* Auth hook is empty because in this mode we blindly trust the
-                   standard streams. Limitting access to thoses is explicitly
+                   standard streams. Limiting access to those is explicitly
                    not `nix-daemon`'s responsibility. */
                 processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
             }

From 7a472a76d4dcbbd0eb7832c0bdcb120d32881e8b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Jan 2021 00:05:04 +0100
Subject: [PATCH 669/882] Add 'nix daemon' command

---
 src/nix/command.hh                            |   2 +
 .../nix-daemon.cc => nix/daemon.cc}           | 114 +++++++++++-------
 src/nix/daemon.md                             |  21 ++++
 src/nix/main.cc                               |   3 +
 tests/common.sh.in                            |   2 +-
 5 files changed, 98 insertions(+), 44 deletions(-)
 rename src/{nix-daemon/nix-daemon.cc => nix/daemon.cc} (77%)
 create mode 100644 src/nix/daemon.md

diff --git a/src/nix/command.hh b/src/nix/command.hh
index 3aae57edd..f325cd906 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -13,6 +13,8 @@ namespace nix {
 
 extern std::string programPath;
 
+extern char * * savedArgv;
+
 class EvalState;
 struct Pos;
 class Store;
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc
similarity index 77%
rename from src/nix-daemon/nix-daemon.cc
rename to src/nix/daemon.cc
index 9227369b8..204d4ce6b 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix/daemon.cc
@@ -1,3 +1,4 @@
+#include "command.hh"
 #include "shared.hh"
 #include "local-store.hh"
 #include "remote-store.hh"
@@ -150,7 +151,7 @@ static ref<Store> openUncachedStore()
 }
 
 
-static void daemonLoop(char * * argv)
+static void daemonLoop()
 {
     if (chdir("/") == -1)
         throw SysError("cannot change current directory");
@@ -232,9 +233,9 @@ static void daemonLoop(char * * argv)
                 setSigChldAction(false);
 
                 //  For debugging, stuff the pid into argv[1].
-                if (peer.pidKnown && argv[1]) {
+                if (peer.pidKnown && savedArgv[1]) {
                     string processName = std::to_string(peer.pid);
-                    strncpy(argv[1], processName.c_str(), strlen(argv[1]));
+                    strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1]));
                 }
 
                 //  Handle the connection.
@@ -264,6 +265,48 @@ static void daemonLoop(char * * argv)
     }
 }
 
+static void runDaemon(bool stdio)
+{
+    if (stdio) {
+        if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
+            auto conn = store->openConnectionWrapper();
+            int from = conn->from.fd;
+            int to = conn->to.fd;
+
+            auto nfds = std::max(from, STDIN_FILENO) + 1;
+            while (true) {
+                fd_set fds;
+                FD_ZERO(&fds);
+                FD_SET(from, &fds);
+                FD_SET(STDIN_FILENO, &fds);
+                if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
+                    throw SysError("waiting for data from client or server");
+                if (FD_ISSET(from, &fds)) {
+                    auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+                    if (res == -1)
+                        throw SysError("splicing data from daemon socket to stdout");
+                    else if (res == 0)
+                        throw EndOfFile("unexpected EOF from daemon socket");
+                }
+                if (FD_ISSET(STDIN_FILENO, &fds)) {
+                    auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+                    if (res == -1)
+                        throw SysError("splicing data from stdin to daemon socket");
+                    else if (res == 0)
+                        return;
+                }
+            }
+        } else {
+            FdSource from(STDIN_FILENO);
+            FdSink to(STDOUT_FILENO);
+            /* Auth hook is empty because in this mode we blindly trust the
+               standard streams. Limiting access to those is explicitly
+               not `nix-daemon`'s responsibility. */
+            processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
+        }
+    } else
+        daemonLoop();
+}
 
 static int main_nix_daemon(int argc, char * * argv)
 {
@@ -285,49 +328,34 @@ static int main_nix_daemon(int argc, char * * argv)
 
         initPlugins();
 
-        if (stdio) {
-            if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
-                auto conn = store->openConnectionWrapper();
-                int from = conn->from.fd;
-                int to = conn->to.fd;
-
-                auto nfds = std::max(from, STDIN_FILENO) + 1;
-                while (true) {
-                    fd_set fds;
-                    FD_ZERO(&fds);
-                    FD_SET(from, &fds);
-                    FD_SET(STDIN_FILENO, &fds);
-                    if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
-                        throw SysError("waiting for data from client or server");
-                    if (FD_ISSET(from, &fds)) {
-                        auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
-                        if (res == -1)
-                            throw SysError("splicing data from daemon socket to stdout");
-                        else if (res == 0)
-                            throw EndOfFile("unexpected EOF from daemon socket");
-                    }
-                    if (FD_ISSET(STDIN_FILENO, &fds)) {
-                        auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
-                        if (res == -1)
-                            throw SysError("splicing data from stdin to daemon socket");
-                        else if (res == 0)
-                            return 0;
-                    }
-                }
-            } else {
-                FdSource from(STDIN_FILENO);
-                FdSink to(STDOUT_FILENO);
-                /* Auth hook is empty because in this mode we blindly trust the
-                   standard streams. Limiting access to those is explicitly
-                   not `nix-daemon`'s responsibility. */
-                processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
-            }
-        } else {
-            daemonLoop(argv);
-        }
+        runDaemon(stdio);
 
         return 0;
     }
 }
 
 static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon);
+
+struct CmdDaemon : StoreCommand
+{
+    std::string description() override
+    {
+        return "daemon to perform store operations on behalf of non-root clients";
+    }
+
+    Category category() override { return catUtility; }
+
+    std::string doc() override
+    {
+        return
+          #include "daemon.md"
+          ;
+    }
+
+    void run(ref<Store> store) override
+    {
+        runDaemon(false);
+    }
+};
+
+static auto rCmdDaemon = registerCommand2<CmdDaemon>({"daemon"});
diff --git a/src/nix/daemon.md b/src/nix/daemon.md
new file mode 100644
index 000000000..e97016a94
--- /dev/null
+++ b/src/nix/daemon.md
@@ -0,0 +1,21 @@
+R""(
+
+# Example
+
+* Run the daemon in the foreground:
+
+  ```console
+  # nix daemon
+  ```
+
+# Description
+
+This command runs the Nix daemon, which is a required component in
+multi-user Nix installations. It performs build actions and other
+operations on the Nix store on behalf of non-root users. Usually you
+don't run the daemon directly; instead it's managed by a service
+management framework such as `systemd`.
+
+Note that this daemon does not fork into the background.
+
+)""
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 398526020..418396280 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -52,6 +52,7 @@ static bool haveInternet()
 }
 
 std::string programPath;
+char * * savedArgv;
 
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
@@ -232,6 +233,8 @@ static auto rCmdHelp = registerCommand<CmdHelp>("help");
 
 void mainWrapped(int argc, char * * argv)
 {
+    savedArgv = argv;
+
     /* The chroot helper needs to be run before any threads have been
        started. */
     if (argc > 0 && argv[0] == chrootHelperName) {
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 5489c0c44..e3bcab507 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -73,7 +73,7 @@ startDaemon() {
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
     rm -f $NIX_STATE_DIR/daemon-socket/socket
-    nix-daemon &
+    nix daemon &
     for ((i = 0; i < 30; i++)); do
         if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
         sleep 1

From 86a2ceeb986609488be1c6794a8e416df3b90c7b Mon Sep 17 00:00:00 2001
From: Jonathan Ringer <jonringer117@gmail.com>
Date: Wed, 13 Jan 2021 16:56:23 -0800
Subject: [PATCH 670/882] Fix gcc10 build

---
 src/libexpr/flake/flake.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 61aeae543..0786fef3d 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -128,7 +128,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean });
                         break;
                     case nInt:
-                        attrs.emplace(attr.name, attr.value->integer);
+                        attrs.emplace(attr.name, (long unsigned int)attr.value->integer);
                         break;
                     default:
                         throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",

From 7af743470c09b835f910d2e25786c080ccfe52c1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 15 Jan 2021 16:37:41 +0000
Subject: [PATCH 671/882] Make public keys and `requireSigs` local-store
 specific again

Thanks @regnat and @edolstra for catching this and comming up with the
solution.

They way I had generalized those is wrong, because local settings for
non-local stores is confusing default. And due to the nature of C++
inheritance, fixing the defaults is more annoying than it should be.
Additionally, I thought we might just drop the check in the substitution
logic since `Store::addToStore` is now streaming, but @regnat rightfully
pointed out that as it downloads dependencies first, that would still be
too late, and also waste effort on possibly unneeded/unwanted
dependencies.

The simple and correct thing to do is just make a store method for the
boolean logic, keeping all the setting and key stuff the way it was
before. That new method is both used by `LocalStore::addToStore` and the
substitution goal check. Perhaps we might eventually make it fancier,
e.g. sending the ValidPathInfo to remote stores for them to validate,
but this is good enough for now.
---
 src/libstore/build/substitution-goal.cc |  4 +---
 src/libstore/local-store.cc             | 14 ++++++++++++-
 src/libstore/local-store.hh             | 14 +++++++++++++
 src/libstore/misc.cc                    |  9 --------
 src/libstore/store-api.hh               | 28 +++++++++++++------------
 5 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index d16584f65..f3c9040bc 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -142,9 +142,7 @@ void SubstitutionGoal::tryNext()
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
-    if (worker.store.requireSigs
-        && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
+    if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info))
     {
         logWarning({
             .name = "Invalid path signature",
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4f48522c6..d6d74a0b0 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1098,11 +1098,23 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
     }
 }
 
+const PublicKeys & LocalStore::getPublicKeys()
+{
+    auto state(_state.lock());
+    if (!state->publicKeys)
+        state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys());
+    return *state->publicKeys;
+}
+
+bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info)
+{
+    return requireSigs && !info.checkSignatures(*this, getPublicKeys());
+}
 
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
+    if (checkSigs && pathInfoIsTrusted(info))
         throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
 
     addTempRoot(info.path);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 69704d266..9d235ba0a 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -35,6 +35,10 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 {
     using LocalFSStoreConfig::LocalFSStoreConfig;
 
+    Setting<bool> requireSigs{(StoreConfig*) this,
+        settings.requireSigs,
+        "require-sigs", "whether store paths should have a trusted signature on import"};
+
     const std::string name() override { return "Local Store"; }
 };
 
@@ -71,6 +75,8 @@ private:
            minFree but not much below availAfterGC, then there is no
            point in starting a new GC. */
         uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
+
+        std::unique_ptr<PublicKeys> publicKeys;
     };
 
     Sync<State> _state;
@@ -88,6 +94,12 @@ public:
     const Path tempRootsDir;
     const Path fnTempRoots;
 
+private:
+
+    const PublicKeys & getPublicKeys();
+
+public:
+
     // Hack for build-remote.cc.
     PathSet locksHeld;
 
@@ -124,6 +136,8 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
+    bool pathInfoIsTrusted(const ValidPathInfo &) override;
+
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 0d4190a56..ad4dccef9 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -282,13 +282,4 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
 }
 
 
-const PublicKeys & Store::getPublicKeys()
-{
-    auto cryptoState(_cryptoState.lock());
-    if (!cryptoState->publicKeys)
-        cryptoState->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys());
-    return *cryptoState->publicKeys;
-}
-
-
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e6a14afc3..3221cf249 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -189,10 +189,6 @@ struct StoreConfig : public Config
 
     const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
 
-    Setting<bool> requireSigs{this,
-        settings.requireSigs,
-        "require-sigs", "whether store paths should have a trusted signature on import"};
-
     Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
@@ -376,6 +372,21 @@ public:
     void queryPathInfo(const StorePath & path,
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
+    /* Check whether the given valid path info is sufficiently well-formed
+       (e.g. hash content-address or signature) in order to be included in the
+       given store.
+
+       These same checks would be performed in addToStore, but this allows an
+       earlier failure in the case where dependencies need to be added too, but
+       the addToStore wouldn't fail until those dependencies are added. Also,
+       we don't really want to add the dependencies listed in a nar info we
+       don't trust anyyways.
+       */
+    virtual bool pathInfoIsTrusted(const ValidPathInfo &)
+    {
+        return true;
+    }
+
 protected:
 
     virtual void queryPathInfoUncached(const StorePath & path,
@@ -719,20 +730,11 @@ public:
         return toRealPath(printStorePath(storePath));
     }
 
-    const PublicKeys & getPublicKeys();
-
     virtual void createUser(const std::string & userName, uid_t userId)
     { }
 
 protected:
 
-    struct CryptoState
-    {
-        std::unique_ptr<PublicKeys> publicKeys;
-    };
-
-    Sync<CryptoState> _cryptoState;
-
     Stats stats;
 
     /* Unsupported methods. */

From 1e13c79a9165e99be9fccfec8e442d14bb66aef0 Mon Sep 17 00:00:00 2001
From: ryneeverett <ryneeverett@gmail.com>
Date: Sat, 16 Jan 2021 19:11:10 +0000
Subject: [PATCH 672/882] Document expected output of 'nix store ping'.

While interpreting the output is fairly intuitive it would be better to
explicitly specify what a good invocation looks like.

That this isn't completely obvious (or at least causes folks to
second-guess themselves) can be seen in a couple user threads:

- https://discourse.nixos.org/t/nixos-cache-fetching-issue/3575/11
- https://discourse.nixos.org/t/newbie-question-cant-get-trivial-example-of-nixops-to-work-on-my-mac/1125/8
---
 src/nix/ping-store.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md
index 322093091..79b108d9c 100644
--- a/src/nix/ping-store.md
+++ b/src/nix/ping-store.md
@@ -27,4 +27,6 @@ argument `--store` *url*) can be accessed. What this means is
 dependent on the type of the store. For instance, for an SSH store it
 means that Nix can connect to the specified machine.
 
+When the command succeeds a zero exit code is returned with no output.
+
 )""

From 1acbb61696c118712417bcd1c59021cc84650e16 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 17 Jan 2021 19:49:28 +0100
Subject: [PATCH 673/882] Tweak

---
 src/nix/ping-store.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md
index 79b108d9c..8c846791b 100644
--- a/src/nix/ping-store.md
+++ b/src/nix/ping-store.md
@@ -27,6 +27,7 @@ argument `--store` *url*) can be accessed. What this means is
 dependent on the type of the store. For instance, for an SSH store it
 means that Nix can connect to the specified machine.
 
-When the command succeeds a zero exit code is returned with no output.
+If the command succeeds, Nix returns a exit code of 0 and does not
+print any output.
 
 )""

From 9432c170e736a6b506d9b35ced5eccff6422ec50 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 21 Dec 2020 17:12:58 +0100
Subject: [PATCH 674/882] Fix the drv output map for non ca derivations

With the `ca-derivation` experimental features, non-ca derivations used
to have their output paths returned as unknown as long as they weren't
built (because of a mistake in the code that systematically erased the
previous value)
---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c61f34275..ab78f1435 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -914,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
         if (realisation)
             outputs.insert_or_assign(outputName, realisation->outPath);
         else
-            outputs.insert_or_assign(outputName, std::nullopt);
+            outputs.insert({outputName, std::nullopt});
     }
 
     return outputs;

From 11b63740e377202e237b7bc74806b82a1eb8ce11 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 21 Dec 2020 21:26:29 +0100
Subject: [PATCH 675/882] Fix content-addressed flake outputs

Prevent some `nix flake` commands to crash by trying to parse a
placeholder output as a store path
---
 src/nix/installables.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 50e3b29c4..34ee238bf 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -501,7 +501,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
 
         auto drvInfo = DerivationInfo{
             std::move(drvPath),
-            state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
+            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
             attr->getAttr(state->sOutputName)->getString()
         };
 

From ea756b3654931f23839aee9f461a8c891c6ffe43 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Jan 2021 14:38:31 +0100
Subject: [PATCH 676/882] --refresh: Imply setting .narinfo disk cache TTL to 0

---
 src/libstore/nar-info-disk-cache.cc | 6 ++++--
 src/nix/main.cc                     | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 8541cc51f..1d8d2d57e 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -109,8 +109,10 @@ public:
                 SQLiteStmt(state->db,
                     "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
                     .use()
-                    (now - settings.ttlNegativeNarInfoCache)
-                    (now - settings.ttlPositiveNarInfoCache)
+                    // Use a minimum TTL to prevent --refresh from
+                    // nuking the entire disk cache.
+                    (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U))
+                    (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U))
                     .exec();
 
                 debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 418396280..80422bd24 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -330,8 +330,11 @@ void mainWrapped(int argc, char * * argv)
             fileTransferSettings.connectTimeout = 1;
     }
 
-    if (args.refresh)
+    if (args.refresh) {
         settings.tarballTtl = 0;
+        settings.ttlNegativeNarInfoCache = 0;
+        settings.ttlPositiveNarInfoCache = 0;
+    }
 
     args.command->second->prepare();
     args.command->second->run();

From 555940f0659e95de7f890ede48e2faba096b3d6d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Jan 2021 22:50:39 +0100
Subject: [PATCH 677/882] Use enumerate()

---
 src/nix/build.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 4cb8ade08..45f63bb7e 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -58,7 +58,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                for (size_t i = 0; i < buildables.size(); ++i)
+                for (const auto & [i, buildable] : enumerate(buildables)) {
                     std::visit(overloaded {
                         [&](BuildableOpaque bo) {
                             std::string symlink = outLink;
@@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                                 store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
-                    }, buildables[i]);
+                    }, buildable);
+                }
 
         updateProfile(buildables);
 

From bc90252cec9af05b897cf209012d44a9b20ea251 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Jan 2021 23:08:58 +0100
Subject: [PATCH 678/882] nix profile install: Support installing non-flakes

Fixes #4458.
---
 src/nix/profile.cc | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index ca95817d0..765d6866e 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -252,8 +252,28 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                 pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
 
                 manifest.elements.emplace_back(std::move(element));
-            } else
-                throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
+            } else {
+                auto buildables = build(store, Realise::Outputs, {installable}, bmNormal);
+
+                for (auto & buildable : buildables) {
+                    ProfileElement element;
+
+                    std::visit(overloaded {
+                        [&](BuildableOpaque bo) {
+                            pathsToBuild.push_back({bo.path, {}});
+                            element.storePaths.insert(bo.path);
+                        },
+                        [&](BuildableFromDrv bfd) {
+                            for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
+                                pathsToBuild.push_back({bfd.drvPath, {output.first}});
+                                element.storePaths.insert(output.second);
+                            }
+                        },
+                    }, buildable);
+
+                    manifest.elements.emplace_back(std::move(element));
+                }
+            }
         }
 
         store->buildPaths(pathsToBuild);

From 259100332f96250d6615d5839f6a77798c77aefb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Jan 2021 10:29:51 +0100
Subject: [PATCH 679/882] Fix clang build

---
 src/nix/build.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 45f63bb7e..724ce9d79 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -58,7 +58,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                for (const auto & [i, buildable] : enumerate(buildables)) {
+                for (const auto & [_i, buildable] : enumerate(buildables)) {
+                    auto i = _i;
                     std::visit(overloaded {
                         [&](BuildableOpaque bo) {
                             std::string symlink = outLink;

From 144cad906991015e997a6b3e7cc69412eb2b8ab1 Mon Sep 17 00:00:00 2001
From: adisbladis <adisbladis@gmail.com>
Date: Mon, 18 Jan 2021 18:13:07 +0100
Subject: [PATCH 680/882] narinfo: Change NAR URLs to be addressed on the NAR
 hash instead of the compressed hash

This change is to simplify [Trustix](https://github.com/tweag/trustix) indexing and makes it possible to reconstruct this URL regardless of the compression used.

In particular this means that https://github.com/tweag/trustix/blob/7c2e9ca597de233846e0b265fb081626ca6c59d8/contrib/nix/nar/nar.go#L61-L71 can be removed and only the bits that are required to establish trust needs to be published in the Trustix build logs.
---
 src/libstore/binary-cache-store.cc | 6 +-----
 tests/binary-cache.sh              | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4f5f8607d..15163ead5 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -176,11 +176,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;
     narInfo->fileSize = fileSize;
-    narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
+    narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar";
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 355a37d97..937585d6f 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -55,7 +55,7 @@ basicTests
 # Test whether Nix notices if the NAR doesn't match the hash in the NAR info.
 clearStore
 
-nar=$(ls $cacheDir/nar/*.nar.xz | head -n1)
+nar=$(ls $cacheDir/nar/*.nar | head -n1)
 mv $nar $nar.good
 mkdir -p $TEST_ROOT/empty
 nix-store --dump $TEST_ROOT/empty | xz > $nar

From 8d4268d1901452164b3e666f2eb6bd6bf516493b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Jan 2021 00:27:36 +0100
Subject: [PATCH 681/882] Improve error formatting
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Changes:

* The divider lines are gone. These were in practice a bit confusing,
  in particular with --show-trace or --keep-going, since then there
  were multiple lines, suggesting a start/end which wasn't the case.

* Instead, multi-line error messages are now indented to align with
  the prefix (e.g. "error: ").

* The 'description' field is gone since we weren't really using it.

* 'hint' is renamed to 'msg' since it really wasn't a hint.

* The error is now printed *before* the location info.

* The 'name' field is no longer printed since most of the time it
  wasn't very useful since it was just the name of the exception (like
  EvalError). Ideally in the future this would be a unique, easily
  googleable error ID (like rustc).

* "trace:" is now just "…". This assumes error contexts start with
  something like "while doing X".

Example before:

  error: --- AssertionError ---------------------------------------------------------------------------------------- nix
  at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix

       6|
       7|   x = assert false; 1;
        |       ^
       8|

  assertion 'false' failed
  ----------------------------------------------------- show-trace -----------------------------------------------------
  trace: while evaluating the attribute 'x' of the derivation 'hello-2.10'
  at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix

     191|         // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) {
     192|           name = "${attrs.pname}-${attrs.version}";
        |           ^
     193|         } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) {

Example after:

  error: assertion 'false' failed

         at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix

              6|
              7|   x = assert false; 1;
               |       ^
              8|

         … while evaluating the attribute 'x' of the derivation 'hello-2.10'

         at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix

            191|         // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) {
            192|           name = "${attrs.pname}-${attrs.version}";
               |           ^
            193|         } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) {
---
 src/build-remote/build-remote.cc        |  52 ++++----
 src/libexpr/attr-set.hh                 |   2 +-
 src/libexpr/eval-inline.hh              |   4 +-
 src/libexpr/eval.cc                     |  18 +--
 src/libexpr/nixexpr.cc                  |   2 +-
 src/libexpr/nixexpr.hh                  |   2 +-
 src/libexpr/parser.y                    |  30 ++---
 src/libexpr/primops.cc                  |  90 ++++++-------
 src/libexpr/primops/context.cc          |   6 +-
 src/libexpr/primops/fetchMercurial.cc   |   4 +-
 src/libexpr/primops/fetchTree.cc        |   6 +-
 src/libexpr/primops/fromTOML.cc         |   2 +-
 src/libstore/build/derivation-goal.cc   |  41 +++---
 src/libstore/build/substitution-goal.cc |   7 +-
 src/libstore/build/worker.cc            |   5 +-
 src/libstore/builtins/buildenv.cc       |  10 +-
 src/libstore/filetransfer.cc            |  15 +--
 src/libstore/local-store.cc             |  32 +----
 src/libstore/optimise-store.cc          |  17 +--
 src/libstore/sqlite.cc                  |   2 +-
 src/libutil/error.cc                    | 169 ++++++++----------------
 src/libutil/error.hh                    |  19 +--
 src/libutil/logging.cc                  |   7 +-
 src/libutil/serialise.cc                |  22 ++-
 src/libutil/tests/logging.cc            |  35 ++---
 src/nix-build/nix-build.cc              |   7 +-
 src/nix-env/nix-env.cc                  |  20 +--
 src/nix-store/nix-store.cc              |  17 +--
 src/nix/daemon.cc                       |   4 +-
 src/nix/upgrade-nix.cc                  |   5 +-
 src/nix/verify.cc                       |  19 +--
 31 files changed, 249 insertions(+), 422 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8348d8c91..a4cf91858 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -172,13 +172,14 @@ static int main_build_remote(int argc, char * * argv)
                     else
                     {
                         // build the hint template.
-                        string hintstring =  "derivation: %s\nrequired (system, features): (%s, %s)";
-                        hintstring += "\n%s available machines:";
-                        hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+                        string errorText =
+                            "Failed to find a machine for remote build!\n"
+                            "derivation: %s\nrequired (system, features): (%s, %s)";
+                        errorText += "\n%s available machines:";
+                        errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
 
-                        for (unsigned int i = 0; i < machines.size(); ++i) {
-                          hintstring += "\n(%s, %s, %s, %s)";
-                        }
+                        for (unsigned int i = 0; i < machines.size(); ++i)
+                            errorText += "\n(%s, %s, %s, %s)";
 
                         // add the template values.
                         string drvstr;
@@ -187,25 +188,21 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto hint = hintformat(hintstring);
-                        hint
-                          % drvstr
-                          % neededSystem
-                          % concatStringsSep<StringSet>(", ", requiredFeatures)
-                          % machines.size();
+                        auto error = hintformat(errorText);
+                        error
+                            % drvstr
+                            % neededSystem
+                            % concatStringsSep<StringSet>(", ", requiredFeatures)
+                            % machines.size();
 
-                        for (auto & m : machines) {
-                          hint % concatStringsSep<vector<string>>(", ", m.systemTypes)
-                            % m.maxJobs
-                            % concatStringsSep<StringSet>(", ", m.supportedFeatures)
-                            % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
-                        }
+                        for (auto & m : machines)
+                            error
+                                % concatStringsSep<vector<string>>(", ", m.systemTypes)
+                                % m.maxJobs
+                                % concatStringsSep<StringSet>(", ", m.supportedFeatures)
+                                % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
 
-                        logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, {
-                              .name = "Remote build",
-                              .description = "Failed to find a machine for remote build!",
-                              .hint = hint
-                        });
+                        printMsg(canBuildLocally ? lvlChatty : lvlWarn, error);
 
                         std::cerr << "# decline\n";
                     }
@@ -230,12 +227,9 @@ static int main_build_remote(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    logError({
-                        .name = "Remote build",
-                        .hint = hintfmt("cannot build on '%s': %s%s",
-                            bestMachine->storeUri, e.what(),
-                            (msg.empty() ? "" : ": " + msg))
-                    });
+                    printError("cannot build on '%s': %s%s",
+                        bestMachine->storeUri, e.what(),
+                        msg.empty() ? "" : ": " + msg);
                     bestMachine->enabled = false;
                     continue;
                 }
diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 7eaa16c59..6d68e5df3 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -77,7 +77,7 @@ public:
         auto a = get(name);
         if (!a)
             throw Error({
-                .hint = hintfmt("attribute '%s' missing", name),
+                .msg = hintfmt("attribute '%s' missing", name),
                 .errPos = pos
             });
 
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index f6dead6b0..655408cd3 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -10,7 +10,7 @@ namespace nix {
 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
     throw EvalError({
-        .hint = hintfmt(s),
+        .msg = hintfmt(s),
         .errPos = pos
     });
 }
@@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
     throw TypeError({
-        .hint = hintfmt(s, showType(v)),
+        .msg = hintfmt(s, showType(v)),
         .errPos = pos
     });
 }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f3471aac7..7271776eb 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -622,7 +622,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
 {
     throw EvalError({
-        .hint = hintfmt(s, s2),
+        .msg = hintfmt(s, s2),
         .errPos = pos
     });
 }
@@ -635,7 +635,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
 {
     throw EvalError({
-        .hint = hintfmt(s, s2, s3),
+        .msg = hintfmt(s, s2, s3),
         .errPos = pos
     });
 }
@@ -644,7 +644,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
 {
     // p1 is where the error occurred; p2 is a position mentioned in the message.
     throw EvalError({
-        .hint = hintfmt(s, sym, p2),
+        .msg = hintfmt(s, sym, p2),
         .errPos = p1
     });
 }
@@ -652,7 +652,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
     throw TypeError({
-        .hint = hintfmt(s),
+        .msg = hintfmt(s),
         .errPos = pos
     });
 }
@@ -660,7 +660,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
     throw TypeError({
-        .hint = hintfmt(s, fun.showNamePos(), s2),
+        .msg = hintfmt(s, fun.showNamePos(), s2),
         .errPos = pos
     });
 }
@@ -668,7 +668,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
 {
     throw AssertionError({
-        .hint = hintfmt(s, s1),
+        .msg = hintfmt(s, s1),
         .errPos = pos
     });
 }
@@ -676,7 +676,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
 LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
 {
     throw UndefinedVarError({
-        .hint = hintfmt(s, s1),
+        .msg = hintfmt(s, s1),
         .errPos = pos
     });
 }
@@ -684,7 +684,7 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
 LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1))
 {
     throw MissingArgumentError({
-        .hint = hintfmt(s, s1),
+        .msg = hintfmt(s, s1),
         .errPos = pos
     });
 }
@@ -2057,7 +2057,7 @@ void EvalState::printStats()
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError({
-        .hint = hintfmt("cannot coerce %1% to a string", showType()),
+        .msg = hintfmt("cannot coerce %1% to a string", showType()),
         .errPos = pos
     });
 }
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index d5698011f..492b819e7 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -284,7 +284,7 @@ void ExprVar::bindVars(const StaticEnv & env)
        "undefined variable" error now. */
     if (withLevel == -1)
         throw UndefinedVarError({
-            .hint = hintfmt("undefined variable '%1%'", name),
+            .msg = hintfmt("undefined variable '%1%'", name),
             .errPos = pos
         });
     fromWith = true;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 530202ff6..cbe9a45bf 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -239,7 +239,7 @@ struct ExprLambda : Expr
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
-                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                .msg = hintfmt("duplicate formal function argument '%1%'", arg),
                 .errPos = pos
             });
     };
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 85eb05d61..49d995bb9 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -32,7 +32,7 @@ namespace nix {
         Path basePath;
         Symbol file;
         FileOrigin origin;
-        ErrorInfo error;
+        std::optional<ErrorInfo> error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -66,8 +66,8 @@ namespace nix {
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
-         .hint = hintfmt("attribute '%1%' already defined at %2%",
-            showAttrPath(attrPath), prevPos),
+         .msg = hintfmt("attribute '%1%' already defined at %2%",
+             showAttrPath(attrPath), prevPos),
          .errPos = pos
     });
 }
@@ -75,7 +75,7 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
-        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+        .msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
         .errPos = pos
     });
 }
@@ -146,7 +146,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
         throw ParseError({
-            .hint = hintfmt("duplicate formal function argument '%1%'",
+            .msg = hintfmt("duplicate formal function argument '%1%'",
                 formal.name),
             .errPos = pos
         });
@@ -258,7 +258,7 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
     data->error = {
-        .hint = hintfmt(error),
+        .msg = hintfmt(error),
         .errPos = makeCurPos(*loc, data)
     };
 }
@@ -338,7 +338,7 @@ expr_function
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
         throw ParseError({
-            .hint = hintfmt("dynamic attributes not allowed in let"),
+            .msg = hintfmt("dynamic attributes not allowed in let"),
             .errPos = CUR_POS
         });
       $$ = new ExprLet($2, $4);
@@ -418,7 +418,7 @@ expr_simple
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
           throw ParseError({
-              .hint = hintfmt("URL literals are disabled"),
+              .msg = hintfmt("URL literals are disabled"),
               .errPos = CUR_POS
           });
       $$ = new ExprString(data->symbols.create($1));
@@ -491,7 +491,7 @@ attrs
           delete str;
       } else
           throw ParseError({
-              .hint = hintfmt("dynamic attributes not allowed in inherit"),
+              .msg = hintfmt("dynamic attributes not allowed in inherit"),
               .errPos = makeCurPos(@2, data)
           });
     }
@@ -576,7 +576,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
     ParseData data(*this);
     data.origin = origin;
     switch (origin) {
-        case foFile: 
+        case foFile:
             data.file = data.symbols.create(path);
             break;
         case foStdin:
@@ -593,7 +593,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
-    if (res) throw ParseError(data.error);
+    if (res) throw ParseError(data.error.value());
 
     data.result->bindVars(staticEnv);
 
@@ -703,7 +703,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         return corepkgsPrefix + path.substr(4);
 
     throw ThrownError({
-        .hint = hintfmt(evalSettings.pureEval
+        .msg = hintfmt(evalSettings.pureEval
             ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
             : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
             path),
@@ -725,8 +725,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
                         store, resolveUri(elem.second), "source", false).first.storePath) };
         } catch (FileTransferError & e) {
             logWarning({
-                .name = "Entry download",
-                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
             });
             res = { false, "" };
         }
@@ -736,8 +735,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, path };
         else {
             logWarning({
-                .name = "Entry not found",
-                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+                .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };
         }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c73a94f4e..a470ed6df 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -115,7 +115,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
+            .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
             .errPos = pos
         });
     }
@@ -282,7 +282,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .hint = hintfmt(
+            .msg = hintfmt(
                 "cannot import '%1%', since path '%2%' is not valid",
                 path, e.path),
             .errPos = pos
@@ -322,7 +322,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     auto count = args[0]->listSize();
     if (count == 0) {
         throw EvalError({
-            .hint = hintfmt("at least one argument to 'exec' required"),
+            .msg = hintfmt("at least one argument to 'exec' required"),
             .errPos = pos
         });
     }
@@ -336,7 +336,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
+            .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
                 program, e.path),
             .errPos = pos
         });
@@ -551,7 +551,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
         throw EvalError({
-            .hint = hintfmt("attribute 'startSet' required"),
+            .msg = hintfmt("attribute 'startSet' required"),
             .errPos = pos
         });
     state.forceList(*startSet->value, pos);
@@ -565,7 +565,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
         throw EvalError({
-            .hint = hintfmt("attribute 'operator' required"),
+            .msg = hintfmt("attribute 'operator' required"),
             .errPos = pos
         });
     state.forceValue(*op->value, pos);
@@ -587,7 +587,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
             throw EvalError({
-                .hint = hintfmt("attribute 'key' required"),
+                .msg = hintfmt("attribute 'key' required"),
                 .errPos = pos
             });
         state.forceValue(*key->value, pos);
@@ -810,7 +810,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
         throw EvalError({
-            .hint = hintfmt("required attribute 'name' missing"),
+            .msg = hintfmt("required attribute 'name' missing"),
             .errPos = pos
         });
     string drvName;
@@ -859,7 +859,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
             else
                 throw EvalError({
-                    .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
+                    .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
                     .errPos = posDrvName
                 });
         };
@@ -869,7 +869,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
                     throw EvalError({
-                        .hint = hintfmt("duplicate derivation output '%1%'", j),
+                        .msg = hintfmt("duplicate derivation output '%1%'", j),
                         .errPos = posDrvName
                     });
                 /* !!! Check whether j is a valid attribute
@@ -879,14 +879,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                    the resulting set. */
                 if (j == "drv")
                     throw EvalError({
-                        .hint = hintfmt("invalid derivation output name 'drv'" ),
+                        .msg = hintfmt("invalid derivation output name 'drv'" ),
                         .errPos = posDrvName
                     });
                 outputs.insert(j);
             }
             if (outputs.empty())
                 throw EvalError({
-                    .hint = hintfmt("derivation cannot have an empty set of outputs"),
+                    .msg = hintfmt("derivation cannot have an empty set of outputs"),
                     .errPos = posDrvName
                 });
         };
@@ -1007,20 +1007,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Do we have all required attributes? */
     if (drv.builder == "")
         throw EvalError({
-            .hint = hintfmt("required attribute 'builder' missing"),
+            .msg = hintfmt("required attribute 'builder' missing"),
             .errPos = posDrvName
         });
 
     if (drv.platform == "")
         throw EvalError({
-            .hint = hintfmt("required attribute 'system' missing"),
+            .msg = hintfmt("required attribute 'system' missing"),
             .errPos = posDrvName
         });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
         throw EvalError({
-            .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+            .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
             .errPos = posDrvName
         });
 
@@ -1031,7 +1031,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            already content addressed. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error({
-                .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
+                .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"),
                 .errPos = posDrvName
             });
 
@@ -1211,7 +1211,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
         throw EvalError({
-            .hint = hintfmt("path '%1%' is not in the Nix store", path),
+            .msg = hintfmt("path '%1%' is not in the Nix store", path),
             .errPos = pos
         });
     auto path2 = state.store->toStorePath(path).first;
@@ -1247,7 +1247,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .hint = hintfmt(
+            .msg = hintfmt(
                 "cannot check the existence of '%1%', since path '%2%' is not valid",
                 path, e.path),
             .errPos = pos
@@ -1324,7 +1324,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
             .errPos = pos
         });
     }
@@ -1363,7 +1363,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
             throw EvalError({
-                .hint = hintfmt("attribute 'path' missing"),
+                .msg = hintfmt("attribute 'path' missing"),
                 .errPos = pos
             });
 
@@ -1374,7 +1374,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
             throw EvalError({
-                .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
+                .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
                 .errPos = pos
             });
         }
@@ -1400,7 +1400,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     std::optional<HashType> ht = parseHashType(type);
     if (!ht)
       throw Error({
-          .hint = hintfmt("unknown hash type '%1%'", type),
+          .msg = hintfmt("unknown hash type '%1%'", type),
           .errPos = pos
       });
 
@@ -1430,7 +1430,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
             .errPos = pos
         });
     }
@@ -1650,7 +1650,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     for (auto path : context) {
         if (path.at(0) != '/')
             throw EvalError( {
-                .hint = hintfmt(
+                .msg = hintfmt(
                     "in 'toFile': the file named '%1%' must not contain a reference "
                     "to a derivation but contains (%2%)",
                     name, path),
@@ -1801,14 +1801,14 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
         throw EvalError({
-            .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+            .msg = hintfmt("string '%1%' cannot refer to other paths", path),
             .errPos = pos
         });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type() != nFunction)
         throw TypeError({
-            .hint = hintfmt(
+            .msg = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
                 showType(*args[0])),
             .errPos = pos
@@ -1875,7 +1875,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             path = state.coerceToPath(*attr.pos, *attr.value, context);
             if (!context.empty())
                 throw EvalError({
-                    .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+                    .msg = hintfmt("string '%1%' cannot refer to other paths", path),
                     .errPos = *attr.pos
                 });
         } else if (attr.name == state.sName)
@@ -1889,13 +1889,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
             throw EvalError({
-                .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
+                .msg = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
                 .errPos = *attr.pos
             });
     }
     if (path.empty())
         throw EvalError({
-            .hint = hintfmt("'path' required"),
+            .msg = hintfmt("'path' required"),
             .errPos = pos
         });
     if (name.empty())
@@ -2010,7 +2010,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
         throw EvalError({
-            .hint = hintfmt("attribute '%1%' missing", attr),
+            .msg = hintfmt("attribute '%1%' missing", attr),
             .errPos = pos
         });
     // !!! add to stack trace?
@@ -2142,7 +2142,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
             throw TypeError({
-                .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
+                .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
                 .errPos = pos
             });
         string name = state.forceStringNoCtx(*j->value, pos);
@@ -2152,7 +2152,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
                 throw TypeError({
-                    .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
+                    .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
                     .errPos = pos
                 });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
@@ -2258,7 +2258,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     }
     if (!args[0]->isLambda())
         throw TypeError({
-            .hint = hintfmt("'functionArgs' requires a function"),
+            .msg = hintfmt("'functionArgs' requires a function"),
             .errPos = pos
         });
 
@@ -2352,7 +2352,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error({
-            .hint = hintfmt("list index %1% is out of bounds", n),
+            .msg = hintfmt("list index %1% is out of bounds", n),
             .errPos = pos
         });
     state.forceValue(*list.listElems()[n], pos);
@@ -2400,7 +2400,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
         throw Error({
-            .hint = hintfmt("'tail' called on an empty list"),
+            .msg = hintfmt("'tail' called on an empty list"),
             .errPos = pos
         });
 
@@ -2639,7 +2639,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
 
     if (len < 0)
         throw EvalError({
-            .hint = hintfmt("cannot create list of size %1%", len),
+            .msg = hintfmt("cannot create list of size %1%", len),
             .errPos = pos
         });
 
@@ -2890,7 +2890,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     NixFloat f2 = state.forceFloat(*args[1], pos);
     if (f2 == 0)
         throw EvalError({
-            .hint = hintfmt("division by zero"),
+            .msg = hintfmt("division by zero"),
             .errPos = pos
         });
 
@@ -2902,7 +2902,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         /* Avoid division overflow as it might raise SIGFPE. */
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
             throw EvalError({
-                .hint = hintfmt("overflow in integer division"),
+                .msg = hintfmt("overflow in integer division"),
                 .errPos = pos
             });
 
@@ -3033,7 +3033,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
 
     if (start < 0)
         throw EvalError({
-            .hint = hintfmt("negative start position in 'substring'"),
+            .msg = hintfmt("negative start position in 'substring'"),
             .errPos = pos
         });
 
@@ -3084,7 +3084,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     std::optional<HashType> ht = parseHashType(type);
     if (!ht)
         throw Error({
-            .hint = hintfmt("unknown hash type '%1%'", type),
+            .msg = hintfmt("unknown hash type '%1%'", type),
             .errPos = pos
         });
 
@@ -3148,12 +3148,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
         if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
-                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
                 .errPos = pos
             });
         } else {
             throw EvalError({
-                .hint = hintfmt("invalid regular expression '%s'", re),
+                .msg = hintfmt("invalid regular expression '%s'", re),
                 .errPos = pos
             });
         }
@@ -3256,12 +3256,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
         if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
-                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
                 .errPos = pos
             });
         } else {
             throw EvalError({
-                .hint = hintfmt("invalid regular expression '%s'", re),
+                .msg = hintfmt("invalid regular expression '%s'", re),
                 .errPos = pos
             });
         }
@@ -3341,7 +3341,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
         throw EvalError({
-            .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
+            .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
             .errPos = pos
         });
 
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index b570fca31..31cf812b4 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -147,7 +147,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
             throw EvalError({
-                .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                .msg = hintfmt("Context key '%s' is not a store path", i.name),
                 .errPos = *i.pos
             });
         if (!settings.readOnlyMode)
@@ -164,7 +164,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
                     throw EvalError({
-                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                        .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
                         .errPos = *i.pos
                     });
                 }
@@ -177,7 +177,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
                 throw EvalError({
-                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                    .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
                     .errPos = *i.pos
                 });
             }
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 845a1ed1b..4830ebec3 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -38,14 +38,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError({
-                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                    .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
                     .errPos = *attr.pos
                 });
         }
 
         if (url.empty())
             throw EvalError({
-                .hint = hintfmt("'url' argument required"),
+                .msg = hintfmt("'url' argument required"),
                 .errPos = pos
             });
 
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index ab80be2d3..48598acaf 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -115,7 +115,7 @@ static void fetchTree(
 
         if (!attrs.count("type"))
             throw Error({
-                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
                 .errPos = pos
             });
 
@@ -177,14 +177,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError({
-                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+                    .msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
                     .errPos = *attr.pos
                 });
             }
 
         if (!url)
             throw EvalError({
-                .hint = hintfmt("'url' argument required"),
+                .msg = hintfmt("'url' argument required"),
                 .errPos = pos
             });
     } else
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 77bff44ae..4c6682dfd 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -82,7 +82,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
         throw EvalError({
-            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+            .msg = hintfmt("while parsing a TOML string: %s", e.what()),
             .errPos = pos
         });
     }
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 2e74cfd6c..c733ccf08 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -87,8 +87,8 @@ void handleDiffHook(
                 printError(chomp(diffRes.second));
         } catch (Error & error) {
             ErrorInfo ei = error.info();
-            ei.hint = hintfmt("diff hook execution failed: %s",
-                (error.info().hint.has_value() ? error.info().hint->str() : ""));
+            // FIXME: wrap errors.
+            ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str());
             logError(ei);
         }
     }
@@ -439,12 +439,9 @@ void DerivationGoal::repairClosure()
     /* Check each path (slow!). */
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
-        logError({
-            .name = "Corrupt path in closure",
-            .hint = hintfmt(
-                "found corrupted or missing path '%s' in the output closure of '%s'",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
-        });
+        printError(
+            "found corrupted or missing path '%s' in the output closure of '%s'",
+            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
             addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
@@ -877,9 +874,12 @@ void DerivationGoal::buildDone()
                 statusToString(status));
 
             if (!logger->isVerbose() && !logTail.empty()) {
-                msg += (format("; last %d log lines:") % logTail.size()).str();
-                for (auto & line : logTail)
-                    msg += "\n  " + line;
+                msg += fmt(";\nlast %d log lines:\n", logTail.size());
+                for (auto & line : logTail) {
+                    msg += "> ";
+                    msg += line;
+                    msg += "\n";
+                }
             }
 
             if (diskFull)
@@ -1055,12 +1055,9 @@ HookReply DerivationGoal::tryBuildHook()
 
     } catch (SysError & e) {
         if (e.errNo == EPIPE) {
-            logError({
-                .name = "Build hook died",
-                .hint = hintfmt(
-                    "build hook died unexpectedly: %s",
-                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
-            });
+            printError(
+                "build hook died unexpectedly: %s",
+                chomp(drainFD(worker.hook->fromHook.readSide.get())));
             worker.hook = 0;
             return rpDecline;
         } else
@@ -3068,10 +3065,7 @@ void DerivationGoal::registerOutputs()
         auto rewriteOutput = [&]() {
             /* Apply hash rewriting if necessary. */
             if (!outputRewrites.empty()) {
-                logWarning({
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
-                });
+                warn("rewriting hashes in '%1%'; cross fingers", actualPath);
 
                 /* FIXME: this is in-memory. */
                 StringSink sink;
@@ -3359,10 +3353,7 @@ void DerivationGoal::registerOutputs()
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(hint);
 
-                logError({
-                    .name = "Output determinism error",
-                    .hint = hint
-                });
+                printError(hint);
 
                 curRound = nrRounds; // we know enough, bail out early
             }
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index d16584f65..760fd8ab8 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -146,11 +146,8 @@ void SubstitutionGoal::tryNext()
         && !sub->isTrusted
         && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
     {
-        logWarning({
-            .name = "Invalid path signature",
-            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath))
-        });
+        warn("substituter '%s' does not have a valid signature for path '%s'",
+            sub->getUri(), worker.store.printStorePath(storePath));
         tryNext();
         return;
     }
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 6c96a93bd..880a93b15 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -454,10 +454,7 @@ bool Worker::pathContentsGood(const StorePath & path)
     }
     pathContentsGoodCache.insert_or_assign(path, res);
     if (!res)
-        logError({
-            .name = "Corrupted path",
-            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
-        });
+        printError("path '%s' is corrupted or missing!", store.printStorePath(path));
     return res;
 }
 
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 802fb87bc..e88fc687a 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         srcFiles = readDirectory(srcDir);
     } catch (SysError & e) {
         if (e.errNo == ENOTDIR) {
-            logWarning({
-                .name = "Create links - directory",
-                .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
-            });
+            warn("not including '%s' in the user environment because it's not a directory", srcDir);
             return;
         }
         throw;
@@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                 throw SysError("getting status of '%1%'", srcFile);
         } catch (SysError & e) {
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
-                logWarning({
-                    .name = "Create links - skipping symlink",
-                    .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
-                });
+                warn("skipping dangling symlink '%s'", dstFile);
                 continue;
             }
             throw;
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 31b4215a9..677ad44cc 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -632,11 +632,7 @@ struct curlFileTransfer : public FileTransfer
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            logError({
-                .name = "File transfer",
-                .hint = hintfmt("unexpected error in download thread: %s",
-                                e.what())
-            });
+            printError("unexpected error in download thread: %s", e.what());
         }
 
         {
@@ -852,11 +848,10 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<
     // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
     // to print different messages for different verbosity levels. For now
     // we add some heuristics for detecting when we want to show the response.
-    if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
-            err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
-    } else {
-        err.hint = hf;
-    }
+    if (response && (response->size() < 1024 || response->find("<html>") != string::npos))
+        err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
+    else
+        err.msg = hf;
 }
 
 bool isUri(const string & s)
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ab78f1435..f306d8505 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -150,12 +150,7 @@ LocalStore::LocalStore(const Params & params)
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            logError({
-                .name = "'build-users-group' not found",
-                .hint = hintfmt(
-                    "warning: the group '%1%' specified in 'build-users-group' does not exist",
-                    settings.buildUsersGroup)
-            });
+            printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
         else {
             struct stat st;
             if (stat(realStoreDir.c_str(), &st))
@@ -1403,12 +1398,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
             if (hash != link.name) {
-                logError({
-                    .name = "Invalid hash",
-                    .hint = hintfmt(
-                        "link '%s' was modified! expected hash '%s', got '%s'",
-                        linkPath, link.name, hash)
-                });
+                printError("link '%s' was modified! expected hash '%s', got '%s'",
+                    linkPath, link.name, hash);
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
                         printInfo("removed link '%s'", linkPath);
@@ -1441,11 +1432,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    logError({
-                        .name = "Invalid hash - path modified",
-                        .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
-                    });
+                    printError("path '%s' was modified! expected hash '%s', got '%s'",
+                        printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true));
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
@@ -1496,10 +1484,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
     if (!done.insert(pathS).second) return;
 
     if (!isStorePath(pathS)) {
-        logError({
-            .name = "Nix path not found",
-            .hint = hintfmt("path '%s' is not in the Nix store", pathS)
-        });
+        printError("path '%s' is not in the Nix store", pathS);
         return;
     }
 
@@ -1522,10 +1507,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            logError({
-                .name = "Missing path with referrers",
-                .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
-            });
+            printError("path '%s' disappeared, but it still has valid referrers!", pathS);
             if (repair)
                 try {
                     repairPath(path);
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index a0d482ddf..78d587139 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -126,16 +126,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        logWarning({
-            .name = "Suspicious file",
-            .hint = hintfmt("skipping suspicious writable file '%1%'", path)
-        });
+        warn("skipping suspicious writable file '%1%'", path);
         return;
     }
 
     /* This can still happen on top-level files. */
     if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
-        debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2));
+        debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2);
         return;
     }
 
@@ -191,10 +188,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     if (st.st_size != stLink.st_size) {
-        logWarning({
-            .name = "Corrupted link",
-            .hint = hintfmt("removing corrupted link '%1%'", linkPath)
-        });
+        warn("removing corrupted link '%s'", linkPath);
         unlink(linkPath.c_str());
         goto retry;
     }
@@ -229,10 +223,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            logError({
-                .name = "Unlink error",
-                .hint = hintfmt("unable to unlink '%1%'", tempLink)
-            });
+            printError("unable to unlink '%1%'", tempLink);
         if (errno == EMLINK) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index f5935ee5c..447b4179b 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -211,7 +211,7 @@ void handleSQLiteBusy(const SQLiteBusy & e)
         lastWarned = now;
         logWarning({
             .name = "Sqlite busy",
-            .hint = hintfmt(e.what())
+            .msg = hintfmt(e.what())
         });
     }
 
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index e7dc3f1d3..bc5f9e440 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -204,168 +204,109 @@ void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
     }
 }
 
+static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s)
+{
+    std::string res;
+    bool first = true;
+
+    while (!s.empty()) {
+        auto end = s.find('\n');
+        if (!first) res += "\n";
+        res += first ? indentFirst : indentRest;
+        res += s.substr(0, end);
+        first = false;
+        if (end == s.npos) break;
+        s = s.substr(end + 1);
+    }
+
+    return res;
+}
+
 std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
 {
-    auto errwidth = std::max<size_t>(getWindowSize().second, 20);
-    string prefix = "";
-
-    string levelString;
+    std::string prefix;
     switch (einfo.level) {
         case Verbosity::lvlError: {
-            levelString = ANSI_RED;
-            levelString += "error:";
-            levelString += ANSI_NORMAL;
+            prefix = ANSI_RED "error";
+            break;
+        }
+        case Verbosity::lvlNotice: {
+            prefix = ANSI_RED "note";
             break;
         }
         case Verbosity::lvlWarn: {
-            levelString = ANSI_YELLOW;
-            levelString += "warning:";
-            levelString += ANSI_NORMAL;
+            prefix = ANSI_YELLOW "warning";
             break;
         }
         case Verbosity::lvlInfo: {
-            levelString = ANSI_GREEN;
-            levelString += "info:";
-            levelString += ANSI_NORMAL;
+            prefix = ANSI_GREEN "info";
             break;
         }
         case Verbosity::lvlTalkative: {
-            levelString = ANSI_GREEN;
-            levelString += "talk:";
-            levelString += ANSI_NORMAL;
+            prefix = ANSI_GREEN "talk";
             break;
         }
         case Verbosity::lvlChatty: {
-            levelString = ANSI_GREEN;
-            levelString += "chat:";
-            levelString += ANSI_NORMAL;
+            prefix = ANSI_GREEN "chat";
             break;
         }
         case Verbosity::lvlVomit: {
-            levelString = ANSI_GREEN;
-            levelString += "vomit:";
-            levelString += ANSI_NORMAL;
+            prefix = ANSI_GREEN "vomit";
             break;
         }
         case Verbosity::lvlDebug: {
-            levelString = ANSI_YELLOW;
-            levelString += "debug:";
-            levelString += ANSI_NORMAL;
-            break;
-        }
-        default: {
-            levelString = fmt("invalid error level: %1%", einfo.level);
+            prefix = ANSI_YELLOW "debug";
             break;
         }
+        default:
+            assert(false);
     }
 
-    auto ndl = prefix.length()
-        + filterANSIEscapes(levelString, true).length()
-        + 7
-        + einfo.name.length()
-        + einfo.programName.value_or("").length();
-    auto dashwidth = std::max<int>(errwidth - ndl, 3);
-
-    std::string dashes(dashwidth, '-');
-
-    // divider.
-    if (einfo.name != "")
-        out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
-            prefix,
-            levelString,
-            einfo.name,
-            dashes,
-            einfo.programName.value_or(""));
+    // FIXME: show the program name as part of the trace?
+    if (einfo.programName && einfo.programName != ErrorInfo::programName)
+        prefix += fmt(" [%s]:" ANSI_NORMAL " ", einfo.programName.value_or(""));
     else
-        out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
-            prefix,
-            levelString,
-            dashes,
-            einfo.programName.value_or(""));
+        prefix += ":" ANSI_NORMAL " ";
 
-    bool nl = false;  // intersperse newline between sections.
-    if (einfo.errPos.has_value() && (*einfo.errPos)) {
-        out << prefix << std::endl;
-        printAtPos(prefix, *einfo.errPos, out);
-        nl = true;
-    }
+    std::ostringstream oss;
+    oss << einfo.msg << "\n";
 
-    // description
-    if (einfo.description != "") {
-        if (nl)
-            out << std::endl << prefix;
-        out << std::endl << prefix << einfo.description;
-        nl = true;
-    }
+    if (einfo.errPos.has_value() && *einfo.errPos) {
+        oss << "\n";
+        printAtPos("", *einfo.errPos, oss);
 
-    if (einfo.errPos.has_value() && (*einfo.errPos)) {
         auto loc = getCodeLines(*einfo.errPos);
 
         // lines of code.
         if (loc.has_value()) {
-            if (nl)
-                out << std::endl << prefix;
-            printCodeLines(out, prefix, *einfo.errPos, *loc);
-            nl = true;
+            oss << "\n";
+            printCodeLines(oss, "", *einfo.errPos, *loc);
+            oss << "\n";
         }
     }
 
-    // hint
-    if (einfo.hint.has_value()) {
-        if (nl)
-            out << std::endl << prefix;
-        out << std::endl << prefix << *einfo.hint;
-        nl = true;
-    }
-
     // traces
-    if (showTrace && !einfo.traces.empty())
-    {
-        const string tracetitle(" show-trace ");
-
-        int fill = errwidth - tracetitle.length();
-        int lw = 0;
-        int rw = 0;
-        const int min_dashes = 3;
-        if (fill > min_dashes * 2) {
-            if (fill % 2 != 0) {
-                lw = fill / 2;
-                rw = lw + 1;
-            }
-            else
-            {
-                lw = rw = fill / 2;
-            }
-        }
-        else
-            lw = rw = min_dashes;
-
-        if (nl)
-            out << std::endl << prefix;
-
-        out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
-
-        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
-        {
-            out << std::endl << prefix;
-            out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
+    if (showTrace && !einfo.traces.empty()) {
+        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {
+            oss << "\n" << "… " << iter->hint.str() << "\n";
 
             if (iter->pos.has_value() && (*iter->pos)) {
                 auto pos = iter->pos.value();
-                out << std::endl << prefix;
-                printAtPos(prefix, pos, out);
+                oss << "\n";
+                printAtPos("", pos, oss);
 
                 auto loc = getCodeLines(pos);
-                if (loc.has_value())
-                {
-                    out << std::endl << prefix;
-                    printCodeLines(out, prefix, pos, *loc);
-                    out << std::endl << prefix;
+                if (loc.has_value()) {
+                    oss << "\n";
+                    printCodeLines(oss, "", pos, *loc);
+                    oss << "\n";
                 }
             }
         }
     }
 
+    out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str()));
+
     return out;
 }
 }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 1e0bde7ea..ff58d3e00 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -107,9 +107,8 @@ struct Trace {
 
 struct ErrorInfo {
     Verbosity level;
-    string name;
-    string description; // FIXME: remove? it seems to be barely used
-    std::optional<hintformat> hint;
+    string name; // FIXME: rename
+    hintformat msg;
     std::optional<ErrPos> errPos;
     std::list<Trace> traces;
 
@@ -133,23 +132,17 @@ public:
 
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
-        : err {.level = lvlError,
-            .hint = hintfmt(args...)
-            }
+        : err { .level = lvlError, .msg = hintfmt(args...) }
         , status(status)
     { }
 
     template<typename... Args>
     BaseError(const std::string & fs, const Args & ... args)
-        : err {.level = lvlError,
-            .hint = hintfmt(fs, args...)
-            }
+        : err { .level = lvlError, .msg = hintfmt(fs, args...) }
     { }
 
     BaseError(hintformat hint)
-        : err {.level = lvlError,
-            .hint = hint
-            }
+        : err { .level = lvlError, .msg = hint }
     { }
 
     BaseError(ErrorInfo && e)
@@ -206,7 +199,7 @@ public:
     {
         errNo = errno;
         auto hf = hintfmt(args...);
-        err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
+        err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
     }
 
     virtual const char* sname() const override { return "SysError"; }
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 6fd0dacef..d2e801175 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -184,7 +184,7 @@ struct JSONLogger : Logger {
         json["action"] = "msg";
         json["level"] = ei.level;
         json["msg"] = oss.str();
-        json["raw_msg"] = ei.hint->str();
+        json["raw_msg"] = ei.msg.str();
 
         if (ei.errPos.has_value() && (*ei.errPos)) {
             json["line"] = ei.errPos->line;
@@ -305,10 +305,7 @@ bool handleJSONLogMessage(const std::string & msg,
         }
 
     } catch (std::exception & e) {
-        logError({
-            .name = "JSON log message",
-            .hint = hintfmt("bad log message from builder: %s", e.what())
-        });
+        printError("bad JSON log message from builder: %s", e.what());
     }
 
     return true;
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 87c1099a1..d1a16b6ba 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -52,10 +52,7 @@ size_t threshold = 256 * 1024 * 1024;
 
 static void warnLargeDump()
 {
-    logWarning({
-        .name = "Large path",
-        .description = "dumping very large path (> 256 MiB); this may run out of memory"
-    });
+    warn("dumping very large path (> 256 MiB); this may run out of memory");
 }
 
 
@@ -306,8 +303,7 @@ Sink & operator << (Sink & sink, const Error & ex)
         << "Error"
         << info.level
         << info.name
-        << info.description
-        << (info.hint ? info.hint->str() : "")
+        << info.msg.str()
         << 0 // FIXME: info.errPos
         << info.traces.size();
     for (auto & trace : info.traces) {
@@ -374,12 +370,14 @@ Error readError(Source & source)
 {
     auto type = readString(source);
     assert(type == "Error");
-    ErrorInfo info;
-    info.level = (Verbosity) readInt(source);
-    info.name = readString(source);
-    info.description = readString(source);
-    auto hint = readString(source);
-    if (hint != "") info.hint = hintformat(std::move(format("%s") % hint));
+    auto level = (Verbosity) readInt(source);
+    auto name = readString(source);
+    auto msg = readString(source);
+    ErrorInfo info {
+        .level = level,
+        .name = name,
+        .msg = hintformat(std::move(format("%s") % msg)),
+    };
     auto havePos = readNum<size_t>(source);
     assert(havePos == 0);
     auto nrTraces = readNum<size_t>(source);
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 5b32c84a4..d990e5499 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -1,3 +1,5 @@
+#if 0
+
 #include "logging.hh"
 #include "nixexpr.hh"
 #include "util.hh"
@@ -41,8 +43,7 @@ namespace nix {
 
         makeJSONLogger(*logger)->logEI({
                 .name = "error name",
-                .description = "error without any code lines.",
-                .hint = hintfmt("this hint has %1% templated %2%!!",
+                .msg = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
                 .errPos = Pos(foFile, problem_file, 02, 13)
@@ -62,7 +63,7 @@ namespace nix {
             throw TestError(e.info());
         } catch (Error &e) {
             ErrorInfo ei = e.info();
-            ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : ""));
+            ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str()));
 
             testing::internal::CaptureStderr();
             logger->logEI(ei);
@@ -95,7 +96,6 @@ namespace nix {
 
         logger->logEI({ .level = lvlInfo,
                         .name = "Info name",
-                        .description = "Info description",
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -109,7 +109,6 @@ namespace nix {
 
         logger->logEI({ .level = lvlTalkative,
                         .name = "Talkative name",
-                        .description = "Talkative description",
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -123,7 +122,6 @@ namespace nix {
 
         logger->logEI({ .level = lvlChatty,
                         .name = "Chatty name",
-                        .description = "Talkative description",
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -137,7 +135,6 @@ namespace nix {
 
         logger->logEI({ .level = lvlDebug,
                         .name = "Debug name",
-                        .description = "Debug description",
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -151,7 +148,6 @@ namespace nix {
 
         logger->logEI({ .level = lvlVomit,
                         .name = "Vomit name",
-                        .description = "Vomit description",
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -167,7 +163,6 @@ namespace nix {
 
         logError({
                 .name = "name",
-                .description = "error description",
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -182,8 +177,7 @@ namespace nix {
 
         logError({
                 .name = "error name",
-                .description = "error with code lines",
-                .hint = hintfmt("this hint has %1% templated %2%!!",
+                .msg = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
                 .errPos = Pos(foString, problem_file, 02, 13),
@@ -200,8 +194,7 @@ namespace nix {
 
         logError({
                 .name = "error name",
-                .description = "error without any code lines.",
-                .hint = hintfmt("this hint has %1% templated %2%!!",
+                .msg = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
                 .errPos = Pos(foFile, problem_file, 02, 13)
@@ -216,7 +209,7 @@ namespace nix {
 
         logError({
                 .name = "error name",
-                .hint = hintfmt("hint %1%", "only"),
+                .msg = hintfmt("hint %1%", "only"),
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -233,8 +226,7 @@ namespace nix {
 
         logWarning({
                 .name = "name",
-                .description = "warning description",
-                .hint = hintfmt("there was a %1%", "warning"),
+                .msg = hintfmt("there was a %1%", "warning"),
             });
 
         auto str = testing::internal::GetCapturedStderr();
@@ -250,8 +242,7 @@ namespace nix {
 
         logWarning({
                 .name = "warning name",
-                .description = "warning description",
-                .hint = hintfmt("this hint has %1% templated %2%!!",
+                .msg = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
                 .errPos = Pos(foStdin, problem_file, 2, 13),
@@ -274,8 +265,7 @@ namespace nix {
 
         auto e = AssertionError(ErrorInfo {
                 .name = "wat",
-                .description = "show-traces",
-                .hint = hintfmt("it has been %1% days since our last error", "zero"),
+                .msg = hintfmt("it has been %1% days since our last error", "zero"),
                 .errPos = Pos(foString, problem_file, 2, 13),
             });
 
@@ -301,8 +291,7 @@ namespace nix {
 
         auto e = AssertionError(ErrorInfo {
                 .name = "wat",
-                .description = "hide traces",
-                .hint = hintfmt("it has been %1% days since our last error", "zero"),
+                .msg = hintfmt("it has been %1% days since our last error", "zero"),
                 .errPos = Pos(foString, problem_file, 2, 13),
             });
 
@@ -377,3 +366,5 @@ namespace nix {
     }
 
 }
+
+#endif
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 38048da52..d1c14596c 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -369,11 +369,8 @@ static void main_nix_build(int argc, char * * argv)
                 shell = drv->queryOutPath() + "/bin/bash";
 
             } catch (Error & e) {
-                logWarning({
-                    .name = "bashInteractive",
-                    .hint = hintfmt("%s; will use bash from your environment",
-                        (e.info().hint ? e.info().hint->str() : ""))
-                });
+                logError(e.info());
+                notice("will use bash from your environment");
                 shell = "bash";
             }
         }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 9963f05d9..d6a16999f 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -124,10 +124,7 @@ static void getAllExprs(EvalState & state,
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
             if (!attrs.insert(attrName).second) {
-                logError({
-                    .name = "Name collision",
-                    .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2)
-                });
+                printError("warning: name collision in input Nix expressions, skipping '%1%'", path2);
                 continue;
             }
             /* Load the expression on demand. */
@@ -876,11 +873,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
             auto placeholder = metaObj.placeholder(j);
             Value * v = i.queryMeta(j);
             if (!v) {
-                logError({
-                    .name = "Invalid meta attribute",
-                    .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'",
-                        i.queryName(), j)
-                });
+                printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
                 placeholder.write(nullptr);
             } else {
                 PathSet context;
@@ -1131,12 +1124,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                             attrs2["name"] = j;
                             Value * v = i.queryMeta(j);
                             if (!v)
-                                logError({
-                                    .name = "Invalid meta attribute",
-                                    .hint = hintfmt(
-                                        "derivation '%s' has invalid meta attribute '%s'",
-                                        i.queryName(), j)
-                                });
+                                printError(
+                                    "derivation '%s' has invalid meta attribute '%s'",
+                                    i.queryName(), j);
                             else {
                                 if (v->type() == nString) {
                                     attrs2["type"] = "string";
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index b97f684a4..b7eda5ba6 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -708,10 +708,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
         else throw UsageError("unknown flag '%1%'", i);
 
     if (store->verifyStore(checkContents, repair)) {
-        logWarning({
-            .name = "Store consistency",
-            .description = "not all errors were fixed"
-            });
+        warn("not all store errors were fixed");
         throw Exit(1);
     }
 }
@@ -733,14 +730,10 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
-            logError({
-                .name = "Hash mismatch",
-                .hint = hintfmt(
-                    "path '%s' was modified! expected hash '%s', got '%s'",
-                    store->printStorePath(path),
-                    info->narHash.to_string(Base32, true),
-                    current.first.to_string(Base32, true))
-            });
+            printError("path '%s' was modified! expected hash '%s', got '%s'",
+                store->printStorePath(path),
+                info->narHash.to_string(Base32, true),
+                current.first.to_string(Base32, true));
             status = 1;
         }
     }
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 204d4ce6b..a358cb0d9 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -258,8 +258,8 @@ static void daemonLoop()
             return;
         } catch (Error & error) {
             ErrorInfo ei = error.info();
-            ei.hint = std::optional(hintfmt("error processing connection: %1%",
-                (error.info().hint.has_value() ? error.info().hint->str() : "")));
+            // FIXME: add to trace?
+            ei.msg = hintfmt("error processing connection: %1%", ei.msg.str());
             logError(ei);
         }
     }
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 299ea40aa..9cd567896 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -61,10 +61,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         if (dryRun) {
             stopProgressBar();
-            logWarning({
-                .name = "Version update",
-                .hint = hintfmt("would upgrade to version %s", version)
-            });
+            warn("would upgrade to version %s", version);
             return;
         }
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index b2963cf74..9b04e032a 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -101,14 +101,10 @@ struct CmdVerify : StorePathsCommand
                     if (hash.first != info->narHash) {
                         corrupted++;
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
-                        logError({
-                            .name = "Hash error - path modified",
-                            .hint = hintfmt(
-                                "path '%s' was modified! expected hash '%s', got '%s'",
-                                store->printStorePath(info->path),
-                                info->narHash.to_string(Base32, true),
-                                hash.first.to_string(Base32, true))
-                        });
+                        printError("path '%s' was modified! expected hash '%s', got '%s'",
+                            store->printStorePath(info->path),
+                            info->narHash.to_string(Base32, true),
+                            hash.first.to_string(Base32, true));
                     }
                 }
 
@@ -156,12 +152,7 @@ struct CmdVerify : StorePathsCommand
                     if (!good) {
                         untrusted++;
                         act2.result(resUntrustedPath, store->printStorePath(info->path));
-                        logError({
-                            .name = "Untrusted path",
-                            .hint = hintfmt("path '%s' is untrusted",
-                                store->printStorePath(info->path))
-                        });
-
+                        printError("path '%s' is untrusted", store->printStorePath(info->path));
                     }
 
                 }

From 40608342cb3772a6d2a6c125cc2237b97c028ab4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Jan 2021 00:49:29 +0100
Subject: [PATCH 682/882] Remove trailing whitespace

---
 src/libutil/error.cc | 3 +--
 src/libutil/util.cc  | 2 +-
 src/libutil/util.hh  | 5 +++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index bc5f9e440..ddeb5412a 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -212,8 +212,7 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR
     while (!s.empty()) {
         auto end = s.find('\n');
         if (!first) res += "\n";
-        res += first ? indentFirst : indentRest;
-        res += s.substr(0, end);
+        res += chomp(std::string(first ? indentFirst : indentRest) + std::string(s.substr(0, end)));
         first = false;
         if (end == s.npos) break;
         s = s.substr(end + 1);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index e6b6d287d..89f7b58f8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1249,7 +1249,7 @@ template StringSet tokenizeString(std::string_view s, const string & separators)
 template vector<string> tokenizeString(std::string_view s, const string & separators);
 
 
-string chomp(const string & s)
+string chomp(std::string_view s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");
     return i == string::npos ? "" : string(s, 0, i + 1);
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index ab0bd865a..ad49c65b3 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -373,8 +373,9 @@ template<class C> Strings quoteStrings(const C & c)
 }
 
 
-/* Remove trailing whitespace from a string. */
-string chomp(const string & s);
+/* Remove trailing whitespace from a string. FIXME: return
+   std::string_view. */
+string chomp(std::string_view s);
 
 
 /* Remove whitespace from the start and end of a string. */

From 55849e153e4b28d03bfca1738c415c438c60f9f6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Jan 2021 00:55:59 +0100
Subject: [PATCH 683/882] Change error position formatting

It's now

  at /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix:7:7:

instead of

  at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix

The new format is more standard and clickable.
---
 src/libutil/error.cc | 22 +++++++++-------------
 tests/misc.sh        |  4 ++--
 2 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index ddeb5412a..5d570a75e 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -43,9 +43,9 @@ string showErrPos(const ErrPos & errPos)
 {
     if (errPos.line > 0) {
         if (errPos.column > 0) {
-            return fmt("(%1%:%2%)", errPos.line, errPos.column);
+            return fmt("%d:%d", errPos.line, errPos.column);
         } else {
-            return fmt("(%1%)", errPos.line);
+            return fmt("%d", errPos.line);
         }
     }
     else {
@@ -178,24 +178,20 @@ void printCodeLines(std::ostream & out,
     }
 }
 
-void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
+void printAtPos(const ErrPos & pos, std::ostream & out)
 {
-    if (pos)
-    {
+    if (pos) {
         switch (pos.origin) {
             case foFile: {
-                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
-                    ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
+                out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
                 break;
             }
             case foString: {
-                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
-                    ANSI_BLUE << " from string" << ANSI_NORMAL;
+                out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
                 break;
             }
             case foStdin: {
-                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
-                    ANSI_BLUE << " from stdin" << ANSI_NORMAL;
+                out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
                 break;
             }
             default:
@@ -272,7 +268,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     if (einfo.errPos.has_value() && *einfo.errPos) {
         oss << "\n";
-        printAtPos("", *einfo.errPos, oss);
+        printAtPos(*einfo.errPos, oss);
 
         auto loc = getCodeLines(*einfo.errPos);
 
@@ -292,7 +288,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             if (iter->pos.has_value() && (*iter->pos)) {
                 auto pos = iter->pos.value();
                 oss << "\n";
-                printAtPos("", pos, oss);
+                printAtPos(pos, oss);
 
                 auto loc = getCodeLines(pos);
                 if (loc.has_value()) {
diff --git a/tests/misc.sh b/tests/misc.sh
index a81c9dbb1..2830856ae 100644
--- a/tests/misc.sh
+++ b/tests/misc.sh
@@ -17,10 +17,10 @@ nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
 eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
-echo $eval_arg_res | grep "at: (1:15) from string"
+echo $eval_arg_res | grep "at «string»:1:15:"
 echo $eval_arg_res | grep "infinite recursion encountered"
 
 eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true)
-echo $eval_stdin_res | grep "at: (1:15) from stdin"
+echo $eval_stdin_res | grep "at «stdin»:1:15:"
 echo $eval_stdin_res | grep "infinite recursion encountered"
 

From 0eb22db3116585821096b7b81295d4bbf5550343 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Jan 2021 12:46:22 +0100
Subject: [PATCH 684/882] Fix macOS build

---
 .../resolve-system-dependencies.cc            | 20 ++++---------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index d30227e4e..27cf53a45 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -39,18 +39,12 @@ std::set<std::string> runResolver(const Path & filename)
         throw SysError("statting '%s'", filename);
 
     if (!S_ISREG(st.st_mode)) {
-        logError({
-            .name = "Regular MACH file",
-            .hint = hintfmt("file '%s' is not a regular file", filename)
-        });
+        printError("file '%s' is not a regular MACH binary", filename);
         return {};
     }
 
     if (st.st_size < sizeof(mach_header_64)) {
-        logError({
-            .name = "File too short",
-            .hint = hintfmt("file '%s' is too short for a MACH binary", filename)
-        });
+        printError("file '%s' is too short for a MACH binary", filename);
         return {};
     }
 
@@ -72,19 +66,13 @@ std::set<std::string> runResolver(const Path & filename)
             }
         }
         if (mach64_offset == 0) {
-            logError({
-                .name = "No mach64 blobs",
-                .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename)
-            });
+            printError("could not find any mach64 blobs in file '%1%', continuing...", filename);
             return {};
         }
     } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
         mach64_offset = 0;
     } else {
-        logError({
-            .name = "Magic number",
-            .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic)
-        });
+        printError("Object file has unknown magic number '%1%', skipping it...", magic);
         return {};
     }
 

From d9367a2dd1f2cfe163b9c42e83a0569808ce6fc9 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 21 Jan 2021 17:30:26 +0100
Subject: [PATCH 685/882] scripts/install-nix-from-closure: only show progress
 if a terminal is used

While the progress dots during the copying of the store work fine on a
normal terminal, those look pretty off if the script is run inside a
provisioning script of e.g. `vagrant` or `packer` where `stderr` and
`stdout` are captured:

    default: .
    default: ..
    default: .
    default: .
    default: .

To work around this, the script checks with `-t 0` if it's
running on an actual terminal and doesn't show the progress if that's not
the case.
---
 scripts/install-nix-from-closure.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 6352a8fac..0ee7ce5af 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -166,9 +166,15 @@ fi
 mkdir -p $dest/store
 
 printf "copying Nix to %s..." "${dest}/store" >&2
+# Insert a newline if no progress is shown.
+if [ ! -t 0 ]; then
+  echo ""
+fi
 
 for i in $(cd "$self/store" >/dev/null && echo ./*); do
-    printf "." >&2
+    if [ -t 0 ]; then
+      printf "." >&2
+    fi
     i_tmp="$dest/store/$i.$$"
     if [ -e "$i_tmp" ]; then
         rm -rf "$i_tmp"

From b7bfc7ee52dd425e0156f369eb4c05a62358f912 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jan 2021 14:54:28 +0100
Subject: [PATCH 686/882] Add FIXME

---
 src/libutil/error.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index e7dc3f1d3..2a67a730a 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -61,6 +61,8 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
     if (errPos.origin == foFile) {
         LinesOfCode loc;
         try {
+            // FIXME: when running as the daemon, make sure we don't
+            // open a file to which the client doesn't have access.
             AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
             if (!fd) return {};
 

From 8c07ed1ddad6595cd679181b0b8d78e09fc6d152 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 22 Jan 2021 15:27:55 +0000
Subject: [PATCH 687/882] Improve documentation and test and requested

---
 src/libstore/store-api.hh          | 6 +++---
 tests/binary-cache-build-remote.sh | 5 ++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3221cf249..9e98eb8f9 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -372,9 +372,9 @@ public:
     void queryPathInfo(const StorePath & path,
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
-    /* Check whether the given valid path info is sufficiently well-formed
-       (e.g. hash content-address or signature) in order to be included in the
-       given store.
+    /* Check whether the given valid path info is sufficiently attested, by
+       either being signed by a trusted public key or content-addressed, in
+       order to be included in the given store.
 
        These same checks would be performed in addToStore, but this allows an
        earlier failure in the case where dependencies need to be added too, but
diff --git a/tests/binary-cache-build-remote.sh b/tests/binary-cache-build-remote.sh
index ed51164a4..81cd21a4a 100644
--- a/tests/binary-cache-build-remote.sh
+++ b/tests/binary-cache-build-remote.sh
@@ -7,7 +7,10 @@ clearCacheCache
 (! nix-build --store "file://$cacheDir" dependencies.nix)
 
 # Succeeds with default store as build remote.
-nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix
+outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix)
+
+# Test that the path exactly exists in the destination store.
+nix path-info --store "file://$cacheDir" $outPath
 
 # Succeeds without any build capability because no-op
 nix-build --store "file://$cacheDir" -j0 dependencies.nix

From 53a709535b42197a9abd3fe46406bb186ad6c751 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 22 Jan 2021 10:21:12 -0500
Subject: [PATCH 688/882] Apply suggestions from code review

Thanks!

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/build-remote/build-remote.cc      | 6 +++---
 src/libstore/build/derivation-goal.cc | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 350bd6cef..68af3e966 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -75,11 +75,11 @@ static int main_build_remote(int argc, char * * argv)
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
-        currentLoad = "/current-load";
+        auto currentLoadName = "/current-load";
         if (auto localStore = store.dynamic_pointer_cast<LocalFSStore>())
-            currentLoad = std::string { localStore->stateDir } + currentLoad;
+            currentLoad = std::string { localStore->stateDir } + currentLoadName;
         else
-            currentLoad = settings.nixStateDir + currentLoad;
+            currentLoad = settings.nixStateDir + currentLoadName;
 
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 953e241d8..fa8b99118 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -3291,7 +3291,7 @@ void DerivationGoal::registerOutputs()
 
         auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
         if (!localStoreP)
-            Unsupported("Can only register outputs with local store");
+            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
         auto & localStore = *localStoreP;
 
         if (buildMode == bmCheck) {
@@ -3426,7 +3426,7 @@ void DerivationGoal::registerOutputs()
     {
         auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
         if (!localStoreP)
-            Unsupported("Can only register outputs with local store");
+            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
         auto & localStore = *localStoreP;
 
         ValidPathInfos infos2;

From a76682466062ef2c972d19f259feeef1c46a44a3 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 22 Jan 2021 14:46:40 -0600
Subject: [PATCH 689/882] Handle missing etag in 304 Not Modified response

GitHub now omits the etag, but 304 implies it matches the one we
provided. Just use that one to avoid having an etag-less resource.

Fixes #4469
---
 src/libstore/filetransfer.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 31b4215a9..1b7eae3ec 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -375,6 +375,13 @@ struct curlFileTransfer : public FileTransfer
             else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
             {
                 result.cached = httpStatus == 304;
+
+                // In 2021, GitHub responds to If-None-Match with 304,
+                // but omits ETag. We just use the If-None-Match etag
+                // since 304 implies they are the same.
+                if (httpStatus == 304 && result.etag == "")
+                    result.etag = request.expectedETag;
+
                 act.progress(result.bodySize, result.bodySize);
                 done = true;
                 callback(std::move(result));

From 1ea5f0b66ca43eb1f6c552b59de170d61bcf540c Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 22 Jan 2021 23:19:52 -0600
Subject: [PATCH 690/882] Remove expectedETag assert in tarball.cc

---
 src/libfetchers/tarball.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 56c014a8c..b8d7d2c70 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -64,7 +64,6 @@ DownloadFileResult downloadFile(
 
     if (res.cached) {
         assert(cached);
-        assert(request.expectedETag == res.etag);
         storePath = std::move(cached->storePath);
     } else {
         StringSink sink;

From b159d23800eec55412621a0b3e6c926a1dbb1755 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Jan 2021 14:38:15 +0100
Subject: [PATCH 691/882] Make '--help' do the same as 'help' (i.e. show a
 manpage)

---
 src/libutil/args.cc | 89 ---------------------------------------------
 src/libutil/args.hh | 14 -------
 src/nix/command.cc  |  5 ---
 src/nix/command.hh  |  2 -
 src/nix/main.cc     | 61 +++++++++----------------------
 src/nix/nar.cc      |  5 ---
 src/nix/store.cc    |  5 ---
 7 files changed, 17 insertions(+), 164 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index fb5cb80fb..2f2e4bb96 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -96,41 +96,6 @@ void Args::parseCmdline(const Strings & _cmdline)
     processArgs(pendingArgs, true);
 }
 
-void Args::printHelp(const string & programName, std::ostream & out)
-{
-    std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName);
-    for (auto & exp : expectedArgs) {
-        std::cout << renderLabels({exp.label});
-        // FIXME: handle arity > 1
-        if (exp.handler.arity == ArityAny) std::cout << "...";
-        if (exp.optional) std::cout << "?";
-    }
-    std::cout << "\n";
-
-    auto s = description();
-    if (s != "")
-        std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n";
-
-    if (longFlags.size()) {
-        std::cout << "\n";
-        std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n";
-        printFlags(out);
-    }
-}
-
-void Args::printFlags(std::ostream & out)
-{
-    Table2 table;
-    for (auto & flag : longFlags) {
-        if (hiddenCategories.count(flag.second->category)) continue;
-        table.push_back(std::make_pair(
-                (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : "    ")
-                + "--" + flag.first + renderLabels(flag.second->labels),
-                flag.second->description));
-    }
-    printTable(out, table);
-}
-
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     assert(pos != end);
@@ -331,28 +296,6 @@ Strings argvToStrings(int argc, char * * argv)
     return args;
 }
 
-std::string renderLabels(const Strings & labels)
-{
-    std::string res;
-    for (auto label : labels) {
-        for (auto & c : label) c = std::toupper(c);
-        res += " " ANSI_ITALIC + label + ANSI_NORMAL;
-    }
-    return res;
-}
-
-void printTable(std::ostream & out, const Table2 & table)
-{
-    size_t max = 0;
-    for (auto & row : table)
-        max = std::max(max, filterANSIEscapes(row.first, true).size());
-    for (auto & row : table) {
-        out << "  " << row.first
-            << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ')
-            << row.second << "\n";
-    }
-}
-
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
@@ -376,38 +319,6 @@ MultiCommand::MultiCommand(const Commands & commands)
     categories[Command::catDefault] = "Available commands";
 }
 
-void MultiCommand::printHelp(const string & programName, std::ostream & out)
-{
-    if (command) {
-        command->second->printHelp(programName + " " + command->first, out);
-        return;
-    }
-
-    out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName);
-
-    out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n";
-    printFlags(out);
-
-    std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory;
-
-    for (auto & [name, commandFun] : commands) {
-        auto command = commandFun();
-        commandsByCategory[command->category()].insert_or_assign(name, command);
-    }
-
-    for (auto & [category, commands] : commandsByCategory) {
-        out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]);
-
-        Table2 table;
-        for (auto & [name, command] : commands) {
-            auto descr = command->description();
-            if (!descr.empty())
-                table.push_back(std::make_pair(name, descr));
-        }
-        printTable(out, table);
-    }
-}
-
 bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     if (Args::processFlag(pos, end)) return true;
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 3783bc84f..fda7852cd 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -20,8 +20,6 @@ public:
        wrong. */
     void parseCmdline(const Strings & cmdline);
 
-    virtual void printHelp(const string & programName, std::ostream & out);
-
     /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
@@ -115,8 +113,6 @@ protected:
 
     virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
 
-    virtual void printFlags(std::ostream & out);
-
     /* Positional arguments. */
     struct ExpectedArg
     {
@@ -223,8 +219,6 @@ public:
 
     MultiCommand(const Commands & commands);
 
-    void printHelp(const string & programName, std::ostream & out) override;
-
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
@@ -234,14 +228,6 @@ public:
 
 Strings argvToStrings(int argc, char * * argv);
 
-/* Helper function for rendering argument labels. */
-std::string renderLabels(const Strings & labels);
-
-/* Helper function for printing 2-column tables. */
-typedef std::vector<std::pair<std::string, std::string>> Table2;
-
-void printTable(std::ostream & out, const Table2 & table);
-
 struct Completion {
     std::string completion;
     std::string description;
diff --git a/src/nix/command.cc b/src/nix/command.cc
index ba58c7d6b..20eeefe91 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -27,11 +27,6 @@ nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & p
     return res;
 }
 
-void NixMultiCommand::printHelp(const string & programName, std::ostream & out)
-{
-    MultiCommand::printHelp(programName, out);
-}
-
 nlohmann::json NixMultiCommand::toJSON()
 {
     // FIXME: use Command::toJSON() as well.
diff --git a/src/nix/command.hh b/src/nix/command.hh
index f325cd906..791dd0f1e 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -25,8 +25,6 @@ static constexpr Command::Category catNixInstallation = 102;
 
 struct NixMultiCommand : virtual MultiCommand, virtual Command
 {
-    void printHelp(const string & programName, std::ostream & out) override;
-
     nlohmann::json toJSON() override;
 };
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 80422bd24..77a13c913 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -54,6 +54,8 @@ static bool haveInternet()
 std::string programPath;
 char * * savedArgv;
 
+struct HelpRequested { };
+
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     bool printBuildLogs = false;
@@ -71,22 +73,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         addFlag({
             .longName = "help",
             .description = "Show usage information.",
-            .handler = {[&]() { if (!completions) showHelpAndExit(); }},
-        });
-
-        addFlag({
-            .longName = "help-config",
-            .description = "Show configuration settings.",
-            .handler = {[&]() {
-                std::cout << "The following configuration settings are available:\n\n";
-                Table2 tbl;
-                std::map<std::string, Config::SettingInfo> settings;
-                globalConfig.getSettings(settings);
-                for (const auto & s : settings)
-                    tbl.emplace_back(s.first, s.second.description);
-                printTable(std::cout, tbl);
-                throw Exit();
-            }},
+            .handler = {[&]() { throw HelpRequested(); }},
         });
 
         addFlag({
@@ -154,33 +141,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         return pos;
     }
 
-    void printFlags(std::ostream & out) override
-    {
-        Args::printFlags(out);
-        std::cout <<
-            "\n"
-            "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n"
-            "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n"
-            "--help-config' for a list of configuration settings.\n";
-    }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-
-#if 0
-        out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n";
-#endif
-
-        std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n";
-    }
-
-    void showHelpAndExit()
-    {
-        printHelp(programName, std::cout);
-        throw Exit();
-    }
-
     std::string description() override
     {
         return "a tool for reproducible and declarative configuration management";
@@ -298,6 +258,18 @@ void mainWrapped(int argc, char * * argv)
 
     try {
         args.parseCmdline(argvToStrings(argc, argv));
+    } catch (HelpRequested &) {
+        std::vector<std::string> subcommand;
+        MultiCommand * command = &args;
+        while (command) {
+            if (command && command->command) {
+                subcommand.push_back(command->command->first);
+                command = dynamic_cast<MultiCommand *>(&*command->command->second);
+            } else
+                break;
+        }
+        showHelp(subcommand);
+        return;
     } catch (UsageError &) {
         if (!completions) throw;
     }
@@ -306,7 +278,8 @@ void mainWrapped(int argc, char * * argv)
 
     initPlugins();
 
-    if (!args.command) args.showHelpAndExit();
+    if (!args.command)
+        throw UsageError("no subcommand specified");
 
     if (args.command->first != "repl"
         && args.command->first != "doctor"
diff --git a/src/nix/nar.cc b/src/nix/nar.cc
index 0775d3c25..dbb043d9b 100644
--- a/src/nix/nar.cc
+++ b/src/nix/nar.cc
@@ -28,11 +28,6 @@ struct CmdNar : NixMultiCommand
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto rCmdNar = registerCommand<CmdNar>("nar");
diff --git a/src/nix/store.cc b/src/nix/store.cc
index e91bcc503..44e53c7c7 100644
--- a/src/nix/store.cc
+++ b/src/nix/store.cc
@@ -21,11 +21,6 @@ struct CmdStore : virtual NixMultiCommand
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto rCmdStore = registerCommand<CmdStore>("store");

From a32073e7e839ea92ada602c0a170855a08afc73a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Jan 2021 14:43:16 +0100
Subject: [PATCH 692/882] Add FIXME

---
 src/libexpr/primops/fetchTree.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 48598acaf..27d8ddf35 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -153,6 +153,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
     fetchTree(state, pos, args, v, std::nullopt);
 }
 
+// FIXME: document
 static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
 
 static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,

From 3ba98ba8f08523e60310cf75ec809bd21d0ce977 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Jan 2021 17:15:38 +0100
Subject: [PATCH 693/882] Tell user to run 'nix log' to get full build logs

---
 src/libstore/build/derivation-goal.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 36bbe46d4..179a010d4 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -896,6 +896,8 @@ void DerivationGoal::buildDone()
                     msg += line;
                     msg += "\n";
                 }
+                msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.",
+                    worker.store.printStorePath(drvPath));
             }
 
             if (diskFull)

From 807d963ee8d23e88f09e28365b045d322530c5aa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Jan 2021 18:19:32 +0100
Subject: [PATCH 694/882] Group subcommands by category

---
 doc/manual/generate-manpage.nix | 23 +++++++++++++++++++----
 doc/manual/utils.nix            | 10 +++++++++-
 src/libutil/args.cc             |  5 ++++-
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index c2c748464..30152088d 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -13,12 +13,27 @@ let
     + showSynopsis { inherit command; args = def.args; }
     + (if def.commands or {} != {}
        then
+         let
+           categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
+           listCommands = cmds:
+             concatStrings (map (name:
+               "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n")
+               (attrNames cmds));
+         in
          "where *subcommand* is one of the following:\n\n"
          # FIXME: group by category
-         + concatStrings (map (name:
-           "* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n")
-           (attrNames def.commands))
-         + "\n"
+         + (if length categories > 1
+            then
+              concatStrings (map
+                (cat:
+                  "**${toString cat.description}:**\n\n"
+                  + listCommands (filterAttrs (n: v: v.category == cat) def.commands)
+                  + "\n"
+                ) categories)
+              + "\n"
+            else
+              listCommands def.commands
+              + "\n")
        else "")
     + (if def ? doc
        then def.doc + "\n\n"
diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix
index 50150bf3e..d4b18472f 100644
--- a/doc/manual/utils.nix
+++ b/doc/manual/utils.nix
@@ -1,7 +1,15 @@
 with builtins;
 
-{
+rec {
   splitLines = s: filter (x: !isList x) (split "\n" s);
 
   concatStrings = concatStringsSep "";
+
+  # FIXME: O(n^2)
+  unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) [];
+
+  nameValuePair = name: value: { inherit name value; };
+
+  filterAttrs = pred: set:
+    listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
 }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 2f2e4bb96..6d57e1a34 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -341,7 +341,10 @@ nlohmann::json MultiCommand::toJSON()
     for (auto & [name, commandFun] : commands) {
         auto command = commandFun();
         auto j = command->toJSON();
-        j["category"] = categories[command->category()];
+        auto cat = nlohmann::json::object();
+        cat["id"] = command->category();
+        cat["description"] = categories[command->category()];
+        j["category"] = std::move(cat);
         cmds[name] = std::move(j);
     }
 

From 36c4d6f59247826dde32ad2e6b5a9471a9a1c911 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Jan 2021 19:03:13 +0100
Subject: [PATCH 695/882] Group common options

---
 doc/manual/generate-manpage.nix | 40 ++++++++++++++++++++-------------
 src/libexpr/common-eval-args.cc |  7 ++++++
 src/libmain/common-args.cc      |  6 ++++-
 src/libmain/common-args.hh      | 17 ++++++++++++--
 src/libutil/args.cc             |  3 +--
 src/libutil/args.hh             |  2 +-
 src/nix/command.cc              |  9 +++++++-
 src/nix/command.hh              |  2 ++
 src/nix/installables.cc         | 13 +++++++++++
 src/nix/main.cc                 |  1 +
 src/nix/sigs.cc                 |  6 ++---
 11 files changed, 80 insertions(+), 26 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 30152088d..a563c31f8 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -38,31 +38,39 @@ let
     + (if def ? doc
        then def.doc + "\n\n"
        else "")
-    + (let s = showFlags def.flags; in
+    + (let s = showOptions def.flags; in
        if s != ""
-       then "# Flags\n\n${s}"
+       then "# Options\n\n${s}"
        else "")
   ;
 
   appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
 
-  showFlags = flags:
-    concatStrings
-      (map (longName:
-        let flag = flags.${longName}; in
-        if flag.category or "" != "config"
-        then
-          "  - `--${longName}`"
-          + (if flag ? shortName then " / `-${flag.shortName}`" else "")
-          + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
-          + "  \n"
-          + "    " + flag.description + "\n\n"
-        else "")
-        (attrNames flags));
+  showOptions = flags:
+    let
+      categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues flags)));
+    in
+      concatStrings (map
+        (cat:
+          (if cat != ""
+           then "**${cat}:**\n\n"
+           else "")
+          + concatStrings
+            (map (longName:
+              let
+                flag = flags.${longName};
+              in
+                "  - `--${longName}`"
+                + (if flag ? shortName then " / `-${flag.shortName}`" else "")
+                + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
+                + "  \n"
+                + "    " + flag.description + "\n\n"
+            ) (attrNames (filterAttrs (n: v: v.category == cat) flags))))
+        categories);
 
   showSynopsis =
     { command, args }:
-    "`${command}` [*flags*...] ${concatStringsSep " "
+    "`${command}` [*option*...] ${concatStringsSep " "
       (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
 
   processCommand = { command, def, filename }:
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index ffe782454..aa14bf79b 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -12,9 +12,12 @@ namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
+    auto category = "Common evaluation options";
+
     addFlag({
         .longName = "arg",
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
+        .category = category,
         .labels = {"name", "expr"},
         .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
     });
@@ -22,6 +25,7 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "argstr",
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
+        .category = category,
         .labels = {"name", "string"},
         .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
     });
@@ -30,6 +34,7 @@ MixEvalArgs::MixEvalArgs()
         .longName = "include",
         .shortName = 'I',
         .description = "Add *path* to the list of locations used to look up `<...>` file names.",
+        .category = category,
         .labels = {"path"},
         .handler = {[&](std::string s) { searchPath.push_back(s); }}
     });
@@ -37,6 +42,7 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "impure",
         .description = "Allow access to mutable paths and repositories.",
+        .category = category,
         .handler = {[&]() {
             evalSettings.pureEval = false;
         }},
@@ -45,6 +51,7 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "override-flake",
         .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.",
+        .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
             auto from = parseFlakeRef(_from, absPath("."));
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index bd5573e5d..ff96ee7d5 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -11,18 +11,21 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         .longName = "verbose",
         .shortName = 'v',
         .description = "Increase the logging verbosity level.",
+        .category = loggingCategory,
         .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
     });
 
     addFlag({
         .longName = "quiet",
         .description = "Decrease the logging verbosity level.",
+        .category = loggingCategory,
         .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
     });
 
     addFlag({
         .longName = "debug",
         .description = "Set the logging verbosity level to 'debug'.",
+        .category = loggingCategory,
         .handler = {[]() { verbosity = lvlDebug; }},
     });
 
@@ -52,6 +55,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     addFlag({
         .longName = "log-format",
         .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.",
+        .category = loggingCategory,
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });
@@ -66,7 +70,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         }}
     });
 
-    std::string cat = "config";
+    std::string cat = "Options to override configuration settings";
     globalConfig.convertToArgs(*this, cat);
 
     // Backward compatibility hack: nix-env already had a --system flag.
diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh
index 47f341619..8e53a7361 100644
--- a/src/libmain/common-args.hh
+++ b/src/libmain/common-args.hh
@@ -4,6 +4,9 @@
 
 namespace nix {
 
+//static constexpr auto commonArgsCategory = "Miscellaneous common options";
+static constexpr auto loggingCategory = "Logging-related options";
+
 struct MixCommonArgs : virtual Args
 {
     string programName;
@@ -16,7 +19,12 @@ struct MixDryRun : virtual Args
 
     MixDryRun()
     {
-        mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun);
+        addFlag({
+            .longName = "dry-run",
+            .description = "Show what this command would do without doing it.",
+            //.category = commonArgsCategory,
+            .handler = {&dryRun, true},
+        });
     }
 };
 
@@ -26,7 +34,12 @@ struct MixJSON : virtual Args
 
     MixJSON()
     {
-        mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json);
+        addFlag({
+            .longName = "json",
+            .description = "Produce output in JSON format, suitable for consumption by another program.",
+            //.category = commonArgsCategory,
+            .handler = {&json, true},
+        });
     }
 };
 
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 6d57e1a34..71bae0504 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -195,8 +195,7 @@ nlohmann::json Args::toJSON()
             j["shortName"] = std::string(1, flag->shortName);
         if (flag->description != "")
             j["description"] = flag->description;
-        if (flag->category != "")
-            j["category"] = flag->category;
+        j["category"] = flag->category;
         if (flag->handler.arity != ArityAny)
             j["arity"] = flag->handler.arity;
         if (!flag->labels.empty())
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index fda7852cd..b1020b101 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -91,7 +91,7 @@ protected:
         { }
     };
 
-    /* Flags. */
+    /* Options. */
     struct Flag
     {
         typedef std::shared_ptr<Flag> ptr;
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 20eeefe91..614dee788 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -61,6 +61,7 @@ StorePathsCommand::StorePathsCommand(bool recursive)
         addFlag({
             .longName = "no-recursive",
             .description = "Apply operation to specified paths only.",
+            .category = installablesCategory,
             .handler = {&this->recursive, false},
         });
     else
@@ -68,10 +69,16 @@ StorePathsCommand::StorePathsCommand(bool recursive)
             .longName = "recursive",
             .shortName = 'r',
             .description = "Apply operation to closure of the specified paths.",
+            .category = installablesCategory,
             .handler = {&this->recursive, true},
         });
 
-    mkFlag(0, "all", "Apply the operation to every store path.", &all);
+    addFlag({
+        .longName = "all",
+        .description = "Apply the operation to every store path.",
+        .category = installablesCategory,
+        .handler = {&all, true},
+    });
 }
 
 void StorePathsCommand::run(ref<Store> store)
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 791dd0f1e..ed6980075 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -23,6 +23,8 @@ static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;
 
+static constexpr auto installablesCategory = "Options that change the interpretation of installables";
+
 struct NixMultiCommand : virtual MultiCommand, virtual Command
 {
     nlohmann::json toJSON() override;
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 34ee238bf..4e6bf4a9a 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -58,39 +58,47 @@ void completeFlakeInputPath(
 
 MixFlakeOptions::MixFlakeOptions()
 {
+    auto category = "Common flake-related options";
+
     addFlag({
         .longName = "recreate-lock-file",
         .description = "Recreate the flake's lock file from scratch.",
+        .category = category,
         .handler = {&lockFlags.recreateLockFile, true}
     });
 
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
+        .category = category,
         .handler = {&lockFlags.updateLockFile, false}
     });
 
     addFlag({
         .longName = "no-write-lock-file",
         .description = "Do not write the flake's newly generated lock file.",
+        .category = category,
         .handler = {&lockFlags.writeLockFile, false}
     });
 
     addFlag({
         .longName = "no-registries",
         .description = "Don't allow lookups in the flake registries.",
+        .category = category,
         .handler = {&lockFlags.useRegistries, false}
     });
 
     addFlag({
         .longName = "commit-lock-file",
         .description = "Commit changes to the flake's lock file.",
+        .category = category,
         .handler = {&lockFlags.commitLockFile, true}
     });
 
     addFlag({
         .longName = "update-input",
         .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+        .category = category,
         .labels = {"input-path"},
         .handler = {[&](std::string s) {
             lockFlags.inputUpdates.insert(flake::parseInputPath(s));
@@ -104,6 +112,7 @@ MixFlakeOptions::MixFlakeOptions()
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).",
+        .category = category,
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
             lockFlags.inputOverrides.insert_or_assign(
@@ -115,6 +124,7 @@ MixFlakeOptions::MixFlakeOptions()
     addFlag({
         .longName = "inputs-from",
         .description = "Use the inputs of the specified flake as registry entries.",
+        .category = category,
         .labels = {"flake-url"},
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
@@ -144,6 +154,7 @@ SourceExprCommand::SourceExprCommand()
         .longName = "file",
         .shortName = 'f',
         .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
+        .category = installablesCategory,
         .labels = {"file"},
         .handler = {&file},
         .completer = completePath
@@ -152,6 +163,7 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "expr",
         .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.",
+        .category = installablesCategory,
         .labels = {"expr"},
         .handler = {&expr}
     });
@@ -159,6 +171,7 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "derivation",
         .description = "Operate on the store derivation rather than its outputs.",
+        .category = installablesCategory,
         .handler = {&operateOn, OperateOn::Derivation},
     });
 }
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 77a13c913..58b643cc5 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -80,6 +80,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName = "print-build-logs",
             .shortName = 'L',
             .description = "Print full build logs on standard error.",
+            .category = loggingCategory,
             .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }},
         });
 
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 3445182f2..c64b472b6 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand
         addFlag({
             .longName = "substituter",
             .shortName = 's',
-            .description = "Use signatures from specified store.",
+            .description = "Copy signatures from the specified store.",
             .labels = {"store-uri"},
             .handler = {[&](std::string s) { substituterUris.push_back(s); }},
         });
@@ -24,7 +24,7 @@ struct CmdCopySigs : StorePathsCommand
 
     std::string description() override
     {
-        return "copy path signatures from substituters (like binary caches)";
+        return "copy store path signatures from substituters";
     }
 
     void run(ref<Store> store, StorePaths storePaths) override
@@ -110,7 +110,7 @@ struct CmdSign : StorePathsCommand
 
     std::string description() override
     {
-        return "sign the specified paths";
+        return "sign store paths";
     }
 
     void run(ref<Store> store, StorePaths storePaths) override

From f15f0b8e83051cd95dacb2784b004c8272957f30 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Jan 2021 10:34:59 +0100
Subject: [PATCH 696/882] Update to lowdown 0.7.9

---
 flake.lock | 17 -----------------
 flake.nix  | 14 ++++++--------
 2 files changed, 6 insertions(+), 25 deletions(-)

diff --git a/flake.lock b/flake.lock
index 9f8c788ac..6fe52fbfd 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,21 +1,5 @@
 {
   "nodes": {
-    "lowdown-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1598695561,
-        "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=",
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1602702596,
@@ -33,7 +17,6 @@
     },
     "root": {
       "inputs": {
-        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs"
       }
     }
diff --git a/flake.nix b/flake.nix
index 9addccd63..fedd0e381 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,9 +2,9 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
+  #inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
-  outputs = { self, nixpkgs, lowdown-src }:
+  outputs = { self, nixpkgs }:
 
     let
 
@@ -200,16 +200,14 @@
         };
 
         lowdown = with final; stdenv.mkDerivation {
-          name = "lowdown-0.7.1";
+          name = "lowdown-0.7.9";
 
-          /*
           src = fetchurl {
-            url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.1.tar.gz;
-            hash = "sha512-1daoAQfYD0LdhK6aFhrSQvadjc5GsSPBZw0fJDb+BEHYMBLjqiUl2A7H8N+l0W4YfGKqbsPYSrCy4vct+7U6FQ==";
+            url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz;
+            hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ==";
           };
-          */
 
-          src = lowdown-src;
+          #src = lowdown-src;
 
           outputs = [ "out" "bin" "dev" ];
 

From 6af6e41df06f0a8a3b919b4052b41d09f0a97678 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 26 Jan 2021 06:22:24 -0500
Subject: [PATCH 697/882] Move command plugin interface to libnixcmd

---
 Makefile                                       |  1 +
 src/build-remote/build-remote.cc               |  2 +-
 src/{nix => libcmd}/command.cc                 |  0
 src/{nix => libcmd}/command.hh                 |  0
 src/{nix => libcmd}/installables.cc            |  0
 src/{nix => libcmd}/installables.hh            |  0
 src/{nix => libcmd}/legacy.cc                  |  0
 src/{nix => libcmd}/legacy.hh                  |  0
 src/libcmd/local.mk                            | 15 +++++++++++++++
 src/{nix => libcmd}/markdown.cc                |  0
 src/{nix => libcmd}/markdown.hh                |  0
 src/libcmd/nix-cmd.pc.in                       |  9 +++++++++
 src/nix-build/nix-build.cc                     |  2 +-
 src/nix-channel/nix-channel.cc                 |  2 +-
 src/nix-collect-garbage/nix-collect-garbage.cc |  2 +-
 src/nix-copy-closure/nix-copy-closure.cc       |  2 +-
 src/nix-env/nix-env.cc                         |  2 +-
 src/nix-instantiate/nix-instantiate.cc         |  2 +-
 src/nix-store/nix-store.cc                     |  2 +-
 src/nix/daemon.cc                              |  2 +-
 src/nix/local.mk                               |  4 ++--
 21 files changed, 36 insertions(+), 11 deletions(-)
 rename src/{nix => libcmd}/command.cc (100%)
 rename src/{nix => libcmd}/command.hh (100%)
 rename src/{nix => libcmd}/installables.cc (100%)
 rename src/{nix => libcmd}/installables.hh (100%)
 rename src/{nix => libcmd}/legacy.cc (100%)
 rename src/{nix => libcmd}/legacy.hh (100%)
 create mode 100644 src/libcmd/local.mk
 rename src/{nix => libcmd}/markdown.cc (100%)
 rename src/{nix => libcmd}/markdown.hh (100%)
 create mode 100644 src/libcmd/nix-cmd.pc.in

diff --git a/Makefile b/Makefile
index f80b8bb82..68ec3ab0c 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,7 @@ makefiles = \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
+  src/libcmd/local.mk \
   src/nix/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 17a0a8373..5b8ab3387 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using std::cin;
diff --git a/src/nix/command.cc b/src/libcmd/command.cc
similarity index 100%
rename from src/nix/command.cc
rename to src/libcmd/command.cc
diff --git a/src/nix/command.hh b/src/libcmd/command.hh
similarity index 100%
rename from src/nix/command.hh
rename to src/libcmd/command.hh
diff --git a/src/nix/installables.cc b/src/libcmd/installables.cc
similarity index 100%
rename from src/nix/installables.cc
rename to src/libcmd/installables.cc
diff --git a/src/nix/installables.hh b/src/libcmd/installables.hh
similarity index 100%
rename from src/nix/installables.hh
rename to src/libcmd/installables.hh
diff --git a/src/nix/legacy.cc b/src/libcmd/legacy.cc
similarity index 100%
rename from src/nix/legacy.cc
rename to src/libcmd/legacy.cc
diff --git a/src/nix/legacy.hh b/src/libcmd/legacy.hh
similarity index 100%
rename from src/nix/legacy.hh
rename to src/libcmd/legacy.hh
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
new file mode 100644
index 000000000..ab0e0e43d
--- /dev/null
+++ b/src/libcmd/local.mk
@@ -0,0 +1,15 @@
+libraries += libcmd
+
+libcmd_NAME = libnixcmd
+
+libcmd_DIR := $(d)
+
+libcmd_SOURCES := $(wildcard $(d)/*.cc)
+
+libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+
+libcmd_LDFLAGS = -llowdown
+
+libcmd_LIBS = libstore libutil libexpr libmain libfetchers
+
+$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))
diff --git a/src/nix/markdown.cc b/src/libcmd/markdown.cc
similarity index 100%
rename from src/nix/markdown.cc
rename to src/libcmd/markdown.cc
diff --git a/src/nix/markdown.hh b/src/libcmd/markdown.hh
similarity index 100%
rename from src/nix/markdown.hh
rename to src/libcmd/markdown.hh
diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in
new file mode 100644
index 000000000..1761a9f41
--- /dev/null
+++ b/src/libcmd/nix-cmd.pc.in
@@ -0,0 +1,9 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Package Manager
+Version: @PACKAGE_VERSION@
+Libs: -L${libdir} -lnixcmd
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index d1c14596c..361f9730d 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -17,7 +17,7 @@
 #include "get-drvs.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using namespace std::string_literals;
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 309970df6..57189d557 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -2,7 +2,7 @@
 #include "globals.hh"
 #include "filetransfer.hh"
 #include "store-api.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 #include "fetchers.hh"
 
 #include <fcntl.h>
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 57092b887..c1769790a 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -2,7 +2,7 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "globals.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 #include <iostream>
 #include <cerrno>
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 10990f7b5..ad2e06067 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,6 +1,6 @@
 #include "shared.hh"
 #include "store-api.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 using namespace nix;
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index d6a16999f..106a78fc4 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -14,7 +14,7 @@
 #include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 #include <cerrno>
 #include <ctime>
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 3956fef6d..ea2e85eb0 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -10,7 +10,7 @@
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "common-eval-args.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 #include <map>
 #include <iostream>
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index b7eda5ba6..37191b9e6 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -9,7 +9,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "graphml.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 
 #include <iostream>
 #include <algorithm>
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index a358cb0d9..26006167d 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -8,7 +8,7 @@
 #include "globals.hh"
 #include "derivations.hh"
 #include "finally.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
 #include "daemon.hh"
 
 #include <algorithm>
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 23c08fc86..83b6dd08b 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -14,9 +14,9 @@ nix_SOURCES := \
   $(wildcard src/nix-instantiate/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd
 
-nix_LIBS = libexpr libmain libfetchers libstore libutil
+nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown
 

From d3c428413367a87ab2d27abe9c7f3c379eb12e1c Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 5 Jan 2021 10:01:22 +0100
Subject: [PATCH 698/882] Make the error message for missing outputs more
 useful

Don't only show the name of the output, but also the derivation to which
this output belongs (as otherwise it's very hard to track back what went
wrong)
---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 01e2fcc7b..9da415c42 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -394,7 +394,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
     OutputPathMap result;
     for (auto & [outName, optOutPath] : resp) {
         if (!optOutPath)
-            throw Error("output '%s' has no store path mapped to it", outName);
+            throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path));
         result.insert_or_assign(outName, *optOutPath);
     }
     return result;

From 9da11bac5797c34b7bb2ee99275befe9c9fb6dd9 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 7 Jan 2021 11:21:43 +0100
Subject: [PATCH 699/882] Fix the error message when a dep is missing

Fix a mismatch in the errors thrown when a needed output was missing
from an input derivation that was leading to a wrong and quite misleading error
message
---
 src/libstore/build/derivation-goal.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 2e74cfd6c..656f92cee 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -539,12 +539,12 @@ void DerivationGoal::inputsRealised()
                     if (!optRealizedInput)
                         throw Error(
                             "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
+                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
                     worker.store.computeFSClosure(*optRealizedInput, inputPaths);
                 } else
                     throw Error(
                         "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
+                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
             }
         }
     }

From 8e758d402ba1045c7b8273f8cb1d6d8d917ca52b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Jan 2021 12:06:03 +0100
Subject: [PATCH 700/882] Remove mkFlag()

---
 src/libmain/shared.cc | 14 ++++++++++----
 src/libutil/args.hh   | 21 ---------------------
 src/nix/eval.cc       |  6 +++++-
 src/nix/hash.cc       | 43 +++++++++++++++++++++++++++++++++----------
 src/nix/ls.cc         | 23 ++++++++++++++++++++---
 src/nix/path-info.cc  | 30 ++++++++++++++++++++++++++----
 src/nix/verify.cc     | 13 +++++++++++--
 7 files changed, 105 insertions(+), 45 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 7e27e95c2..5baaff3e9 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -229,11 +229,17 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time");
     intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout");
 
-    mkFlag(0, "readonly-mode", "Do not write to the Nix store.",
-        &settings.readOnlyMode);
+    addFlag({
+        .longName = "readonly-mode",
+        .description = "Do not write to the Nix store.",
+        .handler = {&settings.readOnlyMode, true},
+    });
 
-    mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.",
-        &gcWarning, false);
+    addFlag({
+        .longName = "no-gc-warning",
+        .description = "Disable warnings about not using `--add-root`.",
+        .handler = {&gcWarning, true},
+    });
 
     addFlag({
         .longName = "store",
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index b1020b101..42d8515ef 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -135,27 +135,6 @@ public:
 
     void addFlag(Flag && flag);
 
-    /* Helper functions for constructing flags / positional
-       arguments. */
-
-    void mkFlag(char shortName, const std::string & name,
-        const std::string & description, bool * dest)
-    {
-        mkFlag(shortName, name, description, dest, true);
-    }
-
-    template<class T>
-    void mkFlag(char shortName, const std::string & longName, const std::string & description,
-        T * dest, const T & value)
-    {
-        addFlag({
-            .longName = longName,
-            .shortName = shortName,
-            .description = description,
-            .handler = {[=]() { *dest = value; }}
-        });
-    }
-
     void expectArgs(ExpectedArg && arg)
     {
         expectedArgs.emplace_back(std::move(arg));
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index b5049ac65..65d61e005 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -18,7 +18,11 @@ struct CmdEval : MixJSON, InstallableCommand
 
     CmdEval()
     {
-        mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw);
+        addFlag({
+            .longName = "raw",
+            .description = "Print strings without quotes or escaping.",
+            .handler = {&raw, true},
+        });
 
         addFlag({
             .longName = "apply",
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 79d506ace..4535e4ab0 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -19,18 +19,41 @@ struct CmdHashBase : Command
 
     CmdHashBase(FileIngestionMethod mode) : mode(mode)
     {
-        mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI);
-        mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64);
-        mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32);
-        mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16);
+        addFlag({
+            .longName = "sri",
+            .description = "Print the hash in SRI format.",
+            .handler = {&base, SRI},
+        });
+
+        addFlag({
+            .longName = "base64",
+            .description = "Print the hash in base-64 format.",
+            .handler = {&base, Base64},
+        });
+
+        addFlag({
+            .longName = "base32",
+            .description = "Print the hash in base-32 (Nix-specific) format.",
+            .handler = {&base, Base32},
+        });
+
+        addFlag({
+            .longName = "base16",
+            .description = "Print the hash in base-16 format.",
+            .handler = {&base, Base16},
+        });
+
         addFlag(Flag::mkHashTypeFlag("type", &ht));
+
         #if 0
-        mkFlag()
-            .longName("modulo")
-            .description("Compute the hash modulo specified the string.")
-            .labels({"modulus"})
-            .dest(&modulus);
-        #endif
+        addFlag({
+            .longName = "modulo",
+            .description = "Compute the hash modulo the specified string.",
+            .labels = {"modulus"},
+            .handler = {&modulus},
+        });
+        #endif\
+
         expectArgs({
             .label = "paths",
             .handler = {&paths},
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index c0b1ecb32..c1dc9a95b 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -17,9 +17,26 @@ struct MixLs : virtual Args, MixJSON
 
     MixLs()
     {
-        mkFlag('R', "recursive", "List subdirectories recursively.", &recursive);
-        mkFlag('l', "long", "Show detailed file information.", &verbose);
-        mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory);
+        addFlag({
+            .longName = "recursive",
+            .shortName = 'R',
+            .description = "List subdirectories recursively.",
+            .handler = {&recursive, true},
+        });
+
+        addFlag({
+            .longName = "long",
+            .shortName = 'l',
+            .description = "Show detailed file information.",
+            .handler = {&verbose, true},
+        });
+
+        addFlag({
+            .longName = "directory",
+            .shortName = 'd',
+            .description = "Show directories rather than their contents.",
+            .handler = {&showDirectory, true},
+        });
     }
 
     void listText(ref<FSAccessor> accessor)
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 0fa88f1bf..518cd5568 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -18,10 +18,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
 
     CmdPathInfo()
     {
-        mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize);
-        mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize);
-        mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable);
-        mkFlag(0, "sigs", "Show signatures.", &showSigs);
+        addFlag({
+            .longName = "size",
+            .shortName = 's',
+            .description = "Print the size of the NAR serialisation of each path.",
+            .handler = {&showSize, true},
+        });
+
+        addFlag({
+            .longName = "closure-size",
+            .shortName = 'S',
+            .description = "Print the sum of the sizes of the NAR serialisations of the closure of each path.",
+            .handler = {&showClosureSize, true},
+        });
+
+        addFlag({
+            .longName = "human-readable",
+            .shortName = 'h',
+            .description = "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.",
+            .handler = {&humanReadable, true},
+        });
+
+        addFlag({
+            .longName = "sigs",
+            .description = "Show signatures.",
+            .handler = {&showSigs, true},
+        });
     }
 
     std::string description() override
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 9b04e032a..1721c7f16 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -18,8 +18,17 @@ struct CmdVerify : StorePathsCommand
 
     CmdVerify()
     {
-        mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents);
-        mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust);
+        addFlag({
+            .longName = "no-contents",
+            .description = "Do not verify the contents of each store path.",
+            .handler = {&noContents, true},
+        });
+
+        addFlag({
+            .longName = "no-trust",
+            .description = "Do not verify whether each store path is trusted.",
+            .handler = {&noTrust, true},
+        });
 
         addFlag({
             .longName = "substituter",

From c03f41055de6f885ade7fa7927bf83fb697a3dba Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Jan 2021 14:02:54 +0100
Subject: [PATCH 701/882] Add traces to errors while updating flake lock file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Example:

$ nix build --show-trace
error: unable to download 'https://api.github.com/repos/NixOS/nixpkgs/commits/no-such-branch': HTTP error 422 ('')

       response body:

       {
         "message": "No commit found for SHA: no-such-branch",
         "documentation_url": "https://docs.github.com/rest/reference/repos#get-a-commit"
       }

       … while fetching the input 'github:NixOS/nixpkgs/no-such-branch'

       … while updating the flake input 'nixpkgs'

       … while updating the lock file of flake 'git+file:///home/eelco/Dev/nix'
---
 src/libexpr/flake/flake.cc  | 474 +++++++++++++++++++-----------------
 src/libfetchers/fetchers.cc |   9 +-
 2 files changed, 252 insertions(+), 231 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 0786fef3d..2e94490d4 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -298,284 +298,298 @@ LockedFlake lockFlake(
 
     auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
 
-    // FIXME: symlink attack
-    auto oldLockFile = LockFile::read(
-        flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
+    try {
 
-    debug("old lock file: %s", oldLockFile);
+        // FIXME: symlink attack
+        auto oldLockFile = LockFile::read(
+            flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
 
-    // FIXME: check whether all overrides are used.
-    std::map<InputPath, FlakeInput> overrides;
-    std::set<InputPath> overridesUsed, updatesUsed;
+        debug("old lock file: %s", oldLockFile);
 
-    for (auto & i : lockFlags.inputOverrides)
-        overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+        // FIXME: check whether all overrides are used.
+        std::map<InputPath, FlakeInput> overrides;
+        std::set<InputPath> overridesUsed, updatesUsed;
 
-    LockFile newLockFile;
+        for (auto & i : lockFlags.inputOverrides)
+            overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
 
-    std::vector<FlakeRef> parents;
+        LockFile newLockFile;
 
-    std::function<void(
-        const FlakeInputs & flakeInputs,
-        std::shared_ptr<Node> node,
-        const InputPath & inputPathPrefix,
-        std::shared_ptr<const Node> oldNode)>
-        computeLocks;
+        std::vector<FlakeRef> parents;
 
-    computeLocks = [&](
-        const FlakeInputs & flakeInputs,
-        std::shared_ptr<Node> node,
-        const InputPath & inputPathPrefix,
-        std::shared_ptr<const Node> oldNode)
-    {
-        debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
+        std::function<void(
+            const FlakeInputs & flakeInputs,
+            std::shared_ptr<Node> node,
+            const InputPath & inputPathPrefix,
+            std::shared_ptr<const Node> oldNode)>
+            computeLocks;
 
-        /* Get the overrides (i.e. attributes of the form
-           'inputs.nixops.inputs.nixpkgs.url = ...'). */
-        // FIXME: check this
-        for (auto & [id, input] : flake.inputs) {
-            for (auto & [idOverride, inputOverride] : input.overrides) {
+        computeLocks = [&](
+            const FlakeInputs & flakeInputs,
+            std::shared_ptr<Node> node,
+            const InputPath & inputPathPrefix,
+            std::shared_ptr<const Node> oldNode)
+        {
+            debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
+
+            /* Get the overrides (i.e. attributes of the form
+               'inputs.nixops.inputs.nixpkgs.url = ...'). */
+            // FIXME: check this
+            for (auto & [id, input] : flake.inputs) {
+                for (auto & [idOverride, inputOverride] : input.overrides) {
+                    auto inputPath(inputPathPrefix);
+                    inputPath.push_back(id);
+                    inputPath.push_back(idOverride);
+                    overrides.insert_or_assign(inputPath, inputOverride);
+                }
+            }
+
+            /* Go over the flake inputs, resolve/fetch them if
+               necessary (i.e. if they're new or the flakeref changed
+               from what's in the lock file). */
+            for (auto & [id, input2] : flakeInputs) {
                 auto inputPath(inputPathPrefix);
                 inputPath.push_back(id);
-                inputPath.push_back(idOverride);
-                overrides.insert_or_assign(inputPath, inputOverride);
-            }
-        }
+                auto inputPathS = printInputPath(inputPath);
+                debug("computing input '%s'", inputPathS);
 
-        /* Go over the flake inputs, resolve/fetch them if
-           necessary (i.e. if they're new or the flakeref changed
-           from what's in the lock file). */
-        for (auto & [id, input2] : flakeInputs) {
-            auto inputPath(inputPathPrefix);
-            inputPath.push_back(id);
-            auto inputPathS = printInputPath(inputPath);
-            debug("computing input '%s'", inputPathS);
+                try {
 
-            /* Do we have an override for this input from one of the
-               ancestors? */
-            auto i = overrides.find(inputPath);
-            bool hasOverride = i != overrides.end();
-            if (hasOverride) overridesUsed.insert(inputPath);
-            auto & input = hasOverride ? i->second : input2;
+                    /* Do we have an override for this input from one of the
+                       ancestors? */
+                    auto i = overrides.find(inputPath);
+                    bool hasOverride = i != overrides.end();
+                    if (hasOverride) overridesUsed.insert(inputPath);
+                    auto & input = hasOverride ? i->second : input2;
 
-            /* Resolve 'follows' later (since it may refer to an input
-               path we haven't processed yet. */
-            if (input.follows) {
-                InputPath target;
-                if (hasOverride || input.absolute)
-                    /* 'follows' from an override is relative to the
-                       root of the graph. */
-                    target = *input.follows;
-                else {
-                    /* Otherwise, it's relative to the current flake. */
-                    target = inputPathPrefix;
-                    for (auto & i : *input.follows) target.push_back(i);
-                }
-                debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
-                node->inputs.insert_or_assign(id, target);
-                continue;
-            }
+                    /* Resolve 'follows' later (since it may refer to an input
+                       path we haven't processed yet. */
+                    if (input.follows) {
+                        InputPath target;
+                        if (hasOverride || input.absolute)
+                            /* 'follows' from an override is relative to the
+                               root of the graph. */
+                            target = *input.follows;
+                        else {
+                            /* Otherwise, it's relative to the current flake. */
+                            target = inputPathPrefix;
+                            for (auto & i : *input.follows) target.push_back(i);
+                        }
+                        debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
+                        node->inputs.insert_or_assign(id, target);
+                        continue;
+                    }
 
-            assert(input.ref);
+                    assert(input.ref);
 
-            /* Do we have an entry in the existing lock file? And we
-               don't have a --update-input flag for this input? */
-            std::shared_ptr<LockedNode> oldLock;
+                    /* Do we have an entry in the existing lock file? And we
+                       don't have a --update-input flag for this input? */
+                    std::shared_ptr<LockedNode> oldLock;
 
-            updatesUsed.insert(inputPath);
+                    updatesUsed.insert(inputPath);
 
-            if (oldNode && !lockFlags.inputUpdates.count(inputPath))
-                if (auto oldLock2 = get(oldNode->inputs, id))
-                    if (auto oldLock3 = std::get_if<0>(&*oldLock2))
-                        oldLock = *oldLock3;
+                    if (oldNode && !lockFlags.inputUpdates.count(inputPath))
+                        if (auto oldLock2 = get(oldNode->inputs, id))
+                            if (auto oldLock3 = std::get_if<0>(&*oldLock2))
+                                oldLock = *oldLock3;
 
-            if (oldLock
-                && oldLock->originalRef == *input.ref
-                && !hasOverride)
-            {
-                debug("keeping existing input '%s'", inputPathS);
+                    if (oldLock
+                        && oldLock->originalRef == *input.ref
+                        && !hasOverride)
+                    {
+                        debug("keeping existing input '%s'", inputPathS);
 
-                /* Copy the input from the old lock since its flakeref
-                   didn't change and there is no override from a
-                   higher level flake. */
-                auto childNode = std::make_shared<LockedNode>(
-                    oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
+                        /* Copy the input from the old lock since its flakeref
+                           didn't change and there is no override from a
+                           higher level flake. */
+                        auto childNode = std::make_shared<LockedNode>(
+                            oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
 
-                node->inputs.insert_or_assign(id, childNode);
+                        node->inputs.insert_or_assign(id, childNode);
 
-                /* If we have an --update-input flag for an input
-                   of this input, then we must fetch the flake to
-                   update it. */
-                auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
+                        /* If we have an --update-input flag for an input
+                           of this input, then we must fetch the flake to
+                           update it. */
+                        auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
-                auto hasChildUpdate =
-                    lb != lockFlags.inputUpdates.end()
-                    && lb->size() > inputPath.size()
-                    && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
+                        auto hasChildUpdate =
+                            lb != lockFlags.inputUpdates.end()
+                            && lb->size() > inputPath.size()
+                            && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
 
-                if (hasChildUpdate) {
-                    auto inputFlake = getFlake(
-                        state, oldLock->lockedRef, false, flakeCache);
-                    computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
-                } else {
-                    /* No need to fetch this flake, we can be
-                       lazy. However there may be new overrides on the
-                       inputs of this flake, so we need to check
-                       those. */
-                    FlakeInputs fakeInputs;
+                        if (hasChildUpdate) {
+                            auto inputFlake = getFlake(
+                                state, oldLock->lockedRef, false, flakeCache);
+                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+                        } else {
+                            /* No need to fetch this flake, we can be
+                               lazy. However there may be new overrides on the
+                               inputs of this flake, so we need to check
+                               those. */
+                            FlakeInputs fakeInputs;
 
-                    for (auto & i : oldLock->inputs) {
-                        if (auto lockedNode = std::get_if<0>(&i.second)) {
-                            fakeInputs.emplace(i.first, FlakeInput {
-                                .ref = (*lockedNode)->originalRef,
-                                .isFlake = (*lockedNode)->isFlake,
-                            });
-                        } else if (auto follows = std::get_if<1>(&i.second)) {
-                            fakeInputs.emplace(i.first, FlakeInput {
-                                .follows = *follows,
-                                .absolute = true
-                            });
+                            for (auto & i : oldLock->inputs) {
+                                if (auto lockedNode = std::get_if<0>(&i.second)) {
+                                    fakeInputs.emplace(i.first, FlakeInput {
+                                        .ref = (*lockedNode)->originalRef,
+                                        .isFlake = (*lockedNode)->isFlake,
+                                    });
+                                } else if (auto follows = std::get_if<1>(&i.second)) {
+                                    fakeInputs.emplace(i.first, FlakeInput {
+                                        .follows = *follows,
+                                        .absolute = true
+                                    });
+                                }
+                            }
+
+                            computeLocks(fakeInputs, childNode, inputPath, oldLock);
+                        }
+
+                    } else {
+                        /* We need to create a new lock file entry. So fetch
+                           this input. */
+                        debug("creating new input '%s'", inputPathS);
+
+                        if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
+                            throw Error("cannot update flake input '%s' in pure mode", inputPathS);
+
+                        if (input.isFlake) {
+                            auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
+
+                            /* Note: in case of an --override-input, we use
+                               the *original* ref (input2.ref) for the
+                               "original" field, rather than the
+                               override. This ensures that the override isn't
+                               nuked the next time we update the lock
+                               file. That is, overrides are sticky unless you
+                               use --no-write-lock-file. */
+                            auto childNode = std::make_shared<LockedNode>(
+                                inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
+
+                            node->inputs.insert_or_assign(id, childNode);
+
+                            /* Guard against circular flake imports. */
+                            for (auto & parent : parents)
+                                if (parent == *input.ref)
+                                    throw Error("found circular import of flake '%s'", parent);
+                            parents.push_back(*input.ref);
+                            Finally cleanup([&]() { parents.pop_back(); });
+
+                            /* Recursively process the inputs of this
+                               flake. Also, unless we already have this flake
+                               in the top-level lock file, use this flake's
+                               own lock file. */
+                            computeLocks(
+                                inputFlake.inputs, childNode, inputPath,
+                                oldLock
+                                ? std::dynamic_pointer_cast<const Node>(oldLock)
+                                : LockFile::read(
+                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+                        }
+
+                        else {
+                            auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                                state, *input.ref, lockFlags.useRegistries, flakeCache);
+                            node->inputs.insert_or_assign(id,
+                                std::make_shared<LockedNode>(lockedRef, *input.ref, false));
                         }
                     }
 
-                    computeLocks(fakeInputs, childNode, inputPath, oldLock);
-                }
-
-            } else {
-                /* We need to create a new lock file entry. So fetch
-                   this input. */
-                debug("creating new input '%s'", inputPathS);
-
-                if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
-                    throw Error("cannot update flake input '%s' in pure mode", inputPathS);
-
-                if (input.isFlake) {
-                    auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
-
-                    /* Note: in case of an --override-input, we use
-                       the *original* ref (input2.ref) for the
-                       "original" field, rather than the
-                       override. This ensures that the override isn't
-                       nuked the next time we update the lock
-                       file. That is, overrides are sticky unless you
-                       use --no-write-lock-file. */
-                    auto childNode = std::make_shared<LockedNode>(
-                        inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
-
-                    node->inputs.insert_or_assign(id, childNode);
-
-                    /* Guard against circular flake imports. */
-                    for (auto & parent : parents)
-                        if (parent == *input.ref)
-                            throw Error("found circular import of flake '%s'", parent);
-                    parents.push_back(*input.ref);
-                    Finally cleanup([&]() { parents.pop_back(); });
-
-                    /* Recursively process the inputs of this
-                       flake. Also, unless we already have this flake
-                       in the top-level lock file, use this flake's
-                       own lock file. */
-                    computeLocks(
-                        inputFlake.inputs, childNode, inputPath,
-                        oldLock
-                        ? std::dynamic_pointer_cast<const Node>(oldLock)
-                        : LockFile::read(
-                            inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
-                }
-
-                else {
-                    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-                        state, *input.ref, lockFlags.useRegistries, flakeCache);
-                    node->inputs.insert_or_assign(id,
-                        std::make_shared<LockedNode>(lockedRef, *input.ref, false));
+                } catch (Error & e) {
+                    e.addTrace({}, "while updating the flake input '%s'", inputPathS);
+                    throw;
                 }
             }
-        }
-    };
+        };
 
-    computeLocks(
-        flake.inputs, newLockFile.root, {},
-        lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+        computeLocks(
+            flake.inputs, newLockFile.root, {},
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
 
-    for (auto & i : lockFlags.inputOverrides)
-        if (!overridesUsed.count(i.first))
-            warn("the flag '--override-input %s %s' does not match any input",
-                printInputPath(i.first), i.second);
+        for (auto & i : lockFlags.inputOverrides)
+            if (!overridesUsed.count(i.first))
+                warn("the flag '--override-input %s %s' does not match any input",
+                    printInputPath(i.first), i.second);
 
-    for (auto & i : lockFlags.inputUpdates)
-        if (!updatesUsed.count(i))
-            warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+        for (auto & i : lockFlags.inputUpdates)
+            if (!updatesUsed.count(i))
+                warn("the flag '--update-input %s' does not match any input", printInputPath(i));
 
-    /* Check 'follows' inputs. */
-    newLockFile.check();
+        /* Check 'follows' inputs. */
+        newLockFile.check();
 
-    debug("new lock file: %s", newLockFile);
+        debug("new lock file: %s", newLockFile);
 
-    /* Check whether we need to / can write the new lock file. */
-    if (!(newLockFile == oldLockFile)) {
+        /* Check whether we need to / can write the new lock file. */
+        if (!(newLockFile == oldLockFile)) {
 
-        auto diff = LockFile::diff(oldLockFile, newLockFile);
+            auto diff = LockFile::diff(oldLockFile, newLockFile);
 
-        if (lockFlags.writeLockFile) {
-            if (auto sourcePath = topRef.input.getSourcePath()) {
-                if (!newLockFile.isImmutable()) {
-                    if (settings.warnDirty)
-                        warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
-                } else {
-                    if (!lockFlags.updateLockFile)
-                        throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
+            if (lockFlags.writeLockFile) {
+                if (auto sourcePath = topRef.input.getSourcePath()) {
+                    if (!newLockFile.isImmutable()) {
+                        if (settings.warnDirty)
+                            warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
+                    } else {
+                        if (!lockFlags.updateLockFile)
+                            throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
 
-                    auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+                        auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
 
-                    auto path = *sourcePath + "/" + relPath;
+                        auto path = *sourcePath + "/" + relPath;
 
-                    bool lockFileExists = pathExists(path);
+                        bool lockFileExists = pathExists(path);
 
-                    if (lockFileExists) {
-                        auto s = chomp(diff);
-                        if (s.empty())
-                            warn("updating lock file '%s'", path);
-                        else
-                            warn("updating lock file '%s':\n%s", path, s);
-                    } else
-                        warn("creating lock file '%s'", path);
+                        if (lockFileExists) {
+                            auto s = chomp(diff);
+                            if (s.empty())
+                                warn("updating lock file '%s'", path);
+                            else
+                                warn("updating lock file '%s':\n%s", path, s);
+                        } else
+                            warn("creating lock file '%s'", path);
 
-                    newLockFile.write(path);
+                        newLockFile.write(path);
 
-                    topRef.input.markChangedFile(
-                        (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                        lockFlags.commitLockFile
-                        ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
-                                relPath, lockFileExists ? "Update" : "Add", diff))
-                        : std::nullopt);
+                        topRef.input.markChangedFile(
+                            (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
+                            lockFlags.commitLockFile
+                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
+                                    relPath, lockFileExists ? "Update" : "Add", diff))
+                            : std::nullopt);
 
-                    /* Rewriting the lockfile changed the top-level
-                       repo, so we should re-read it. FIXME: we could
-                       also just clear the 'rev' field... */
-                    auto prevLockedRef = flake.lockedRef;
-                    FlakeCache dummyCache;
-                    flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
+                        /* Rewriting the lockfile changed the top-level
+                           repo, so we should re-read it. FIXME: we could
+                           also just clear the 'rev' field... */
+                        auto prevLockedRef = flake.lockedRef;
+                        FlakeCache dummyCache;
+                        flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
 
-                    if (lockFlags.commitLockFile &&
-                        flake.lockedRef.input.getRev() &&
-                        prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
-                        warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
+                        if (lockFlags.commitLockFile &&
+                            flake.lockedRef.input.getRev() &&
+                            prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
+                            warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
 
-                    /* Make sure that we picked up the change,
-                       i.e. the tree should usually be dirty
-                       now. Corner case: we could have reverted from a
-                       dirty to a clean tree! */
-                    if (flake.lockedRef.input == prevLockedRef.input
-                        && !flake.lockedRef.input.isImmutable())
-                        throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
-                }
+                        /* Make sure that we picked up the change,
+                           i.e. the tree should usually be dirty
+                           now. Corner case: we could have reverted from a
+                           dirty to a clean tree! */
+                        if (flake.lockedRef.input == prevLockedRef.input
+                            && !flake.lockedRef.input.isImmutable())
+                            throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
+                    }
+                } else
+                    throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
             } else
-                throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
-        } else
-            warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
-    }
+                warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
+        }
 
-    return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+        return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+
+    } catch (Error & e) {
+        e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string());
+        throw;
+    }
 }
 
 void callFlake(EvalState & state,
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index e6741a451..916e0a8e8 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -132,7 +132,14 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
         }
     }
 
-    auto [tree, input] = scheme->fetch(store, *this);
+    auto [tree, input] = [&]() -> std::pair<Tree, Input> {
+        try {
+            return scheme->fetch(store, *this);
+        } catch (Error & e) {
+            e.addTrace({}, "while fetching the input '%s'", to_string());
+            throw;
+        }
+    }();
 
     if (tree.actualPath == "")
         tree.actualPath = store->toRealPath(tree.storePath);

From 965dc6070a1b7dc582d90039c670d436f4a2e9f6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Jan 2021 14:04:49 +0100
Subject: [PATCH 702/882] Drop trailing whitespace

---
 src/libstore/filetransfer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 563f49170..8ea5cdc9d 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -856,7 +856,7 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<
     // to print different messages for different verbosity levels. For now
     // we add some heuristics for detecting when we want to show the response.
     if (response && (response->size() < 1024 || response->find("<html>") != string::npos))
-        err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
+        err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
     else
         err.msg = hf;
 }

From 12de0466fea6558ccb0dd5b98b72d7a068c9b5e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Jan 2021 14:46:10 +0100
Subject: [PATCH 703/882] Add trace to build errors during
 import-from-derivation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Example:

error: builder for '/nix/store/9ysqfidhipyzfiy54mh77iqn29j6cpsb-failing.drv' failed with exit code 1;
       last 1 log lines:
       > FAIL
       For full logs, run 'nix log /nix/store/9ysqfidhipyzfiy54mh77iqn29j6cpsb-failing.drv'.

       … while importing '/nix/store/pfp4a4bjh642ylxyipncqs03z6kkgfvy-failing'

       at /nix/store/25wgzr2qrqqiqfbdb1chpiry221cjglc-source/flake.nix:58:15:

           57|
           58|         ifd = import self.hydraJobs.broken;
             |               ^
           59|
---
 src/libexpr/primops.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a470ed6df..13565b950 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -118,6 +118,9 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
             .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
             .errPos = pos
         });
+    } catch (Error & e) {
+        e.addTrace(pos, "while importing '%s'", path);
+        throw e;
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));

From 9355ecd54301372b6a919a2205340f904c7a51c6 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Dec 2020 17:24:30 +0100
Subject: [PATCH 704/882] Add a new Cmd type working on RealisedPaths

Where a `RealisedPath` is a store path with its history, meaning either
an opaque path for stuff that has been directly added to the store, or a
`Realisation` for stuff that has been built by a derivation

This is a low-level refactoring that doesn't bring anything by itself
(except a few dozen extra lines of code :/ ), but raising the
abstraction level a bit is important on a number of levels:

- Commands like `nix build` have to query for the realisations after the
  build is finished which is fragile (see
  27905f12e4a7207450abe37c9ed78e31603b67e1 for example). Having them
  oprate directly at the realisation level would avoid that
- Others like `nix copy` currently operate directly on (built) store
  paths, but need a bit more information as they will need to register
  the realisations on the remote side
---
 src/libcmd/command.cc       | 42 ++++++++++-------
 src/libcmd/command.hh       | 23 ++++++++--
 src/libcmd/installables.cc  | 48 ++++++++++++++++----
 src/libstore/realisation.cc | 31 +++++++++++++
 src/libstore/realisation.hh | 90 +++++++++++++++++++++++++++++++++----
 src/nix/copy.cc             |  2 +
 6 files changed, 200 insertions(+), 36 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 614dee788..efdc98d5a 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -54,7 +54,7 @@ void StoreCommand::run()
     run(getStore());
 }
 
-StorePathsCommand::StorePathsCommand(bool recursive)
+RealisedPathsCommand::RealisedPathsCommand(bool recursive)
     : recursive(recursive)
 {
     if (recursive)
@@ -81,30 +81,40 @@ StorePathsCommand::StorePathsCommand(bool recursive)
     });
 }
 
-void StorePathsCommand::run(ref<Store> store)
+void RealisedPathsCommand::run(ref<Store> store)
 {
-    StorePaths storePaths;
-
+    std::vector<RealisedPath> paths;
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
+        // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            storePaths.push_back(p);
-    }
-
-    else {
-        for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
-            storePaths.push_back(p);
-
+            paths.push_back(p);
+    } else {
+        auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
         if (recursive) {
-            StorePathSet closure;
-            store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false);
-            storePaths.clear();
-            for (auto & p : closure)
-                storePaths.push_back(p);
+            auto roots = std::move(pathSet);
+            pathSet = {};
+            RealisedPath::closure(*store, roots, pathSet);
         }
+        for (auto & path : pathSet)
+            paths.push_back(path);
     }
 
+    run(store, std::move(paths));
+}
+
+StorePathsCommand::StorePathsCommand(bool recursive)
+    : RealisedPathsCommand(recursive)
+{
+}
+
+void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
+{
+    StorePaths storePaths;
+    for (auto & p : paths)
+        storePaths.push_back(p.path());
+
     run(store, std::move(storePaths));
 }
 
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index ed6980075..8c0b3a94a 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -141,7 +141,7 @@ private:
 };
 
 /* A command that operates on zero or more store paths. */
-struct StorePathsCommand : public InstallablesCommand
+struct RealisedPathsCommand : public InstallablesCommand
 {
 private:
 
@@ -154,17 +154,28 @@ protected:
 
 public:
 
-    StorePathsCommand(bool recursive = false);
+    RealisedPathsCommand(bool recursive = false);
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
 
     void run(ref<Store> store) override;
 
     bool useDefaultInstallables() override { return !all; }
 };
 
+struct StorePathsCommand : public RealisedPathsCommand
+{
+    StorePathsCommand(bool recursive = false);
+
+    using RealisedPathsCommand::run;
+
+    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+
+    void run(ref<Store> store, std::vector<RealisedPath> paths) override;
+};
+
 /* A command that operates on exactly one store path. */
 struct StorePathCommand : public InstallablesCommand
 {
@@ -218,6 +229,12 @@ std::set<StorePath> toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables,
     bool useDeriver = false);
 
+std::set<RealisedPath> toRealisedPaths(
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
+    std::vector<std::shared_ptr<Installable>> installables);
+
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */
 Strings editorFor(const Pos & pos);
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 4e6bf4a9a..98a27ded9 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -704,23 +704,43 @@ Buildables build(ref<Store> store, Realise mode,
     return buildables;
 }
 
-StorePathSet toStorePaths(ref<Store> store,
-    Realise mode, OperateOn operateOn,
+std::set<RealisedPath> toRealisedPaths(
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
     std::vector<std::shared_ptr<Installable>> installables)
 {
-    StorePathSet outPaths;
-
+    std::set<RealisedPath> res;
     if (operateOn == OperateOn::Output) {
         for (auto & b : build(store, mode, installables))
             std::visit(overloaded {
                 [&](BuildableOpaque bo) {
-                    outPaths.insert(bo.path);
+                    res.insert(bo.path);
                 },
                 [&](BuildableFromDrv bfd) {
+                    auto drv = store->readDerivation(bfd.drvPath);
+                    auto outputHashes = staticOutputHashes(*store, drv);
                     for (auto & output : bfd.outputs) {
-                        if (!output.second)
-                            throw Error("Cannot operate on output of unbuilt CA drv");
-                        outPaths.insert(*output.second);
+                        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                            if (!outputHashes.count(output.first))
+                                throw Error(
+                                    "The derivation %s doesn't have an output "
+                                    "named %s",
+                                    store->printStorePath(bfd.drvPath),
+                                    output.first);
+                            auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
+                            auto realisation = store->queryRealisation(outputId);
+                            if (!realisation)
+                                throw Error("Cannot operate on output of unbuilt CA drv %s", outputId.to_string());
+                            res.insert(RealisedPath{*realisation});
+                        }
+                        else {
+                            // If ca-derivations isn't enabled, behave as if
+                            // all the paths are opaque to keep the default
+                            // behavior
+                            assert(output.second);
+                            res.insert(*output.second);
+                        }
                     }
                 },
             }, b);
@@ -731,9 +751,19 @@ StorePathSet toStorePaths(ref<Store> store,
         for (auto & i : installables)
             for (auto & b : i->toBuildables())
                 if (auto bfd = std::get_if<BuildableFromDrv>(&b))
-                    outPaths.insert(bfd->drvPath);
+                    res.insert(bfd->drvPath);
     }
 
+    return res;
+}
+
+StorePathSet toStorePaths(ref<Store> store,
+    Realise mode, OperateOn operateOn,
+    std::vector<std::shared_ptr<Installable>> installables)
+{
+    StorePathSet outPaths;
+    for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
+            outPaths.insert(path.path());
     return outPaths;
 }
 
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 47ad90eee..c9b66186f 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -46,4 +46,35 @@ Realisation Realisation::fromJSON(
     };
 }
 
+StorePath RealisedPath::path() const {
+    return visit([](auto && arg) { return arg.getPath(); });
+}
+
+void RealisedPath::closure(
+    Store& store,
+    const RealisedPath::Set& startPaths,
+    RealisedPath::Set& ret)
+{
+    // FIXME: This only builds the store-path closure, not the real realisation
+    // closure
+    StorePathSet initialStorePaths, pathsClosure;
+    for (auto& path : startPaths)
+        initialStorePaths.insert(path.path());
+    store.computeFSClosure(initialStorePaths, pathsClosure);
+    ret.insert(startPaths.begin(), startPaths.end());
+    ret.insert(pathsClosure.begin(), pathsClosure.end());
+}
+
+void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const
+{
+    RealisedPath::closure(store, {*this}, ret);
+}
+
+RealisedPath::Set RealisedPath::closure(Store& store) const
+{
+    RealisedPath::Set ret;
+    closure(store, ret);
+    return ret;
+}
+
 } // namespace nix
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 4b8ead3c5..1ecddc4d1 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -3,6 +3,34 @@
 #include "path.hh"
 #include <nlohmann/json_fwd.hpp>
 
+
+/* Awfull hacky generation of the comparison operators by doing a lexicographic
+ * comparison between the choosen fields
+ * ```
+ * GENERATE_CMP(ClassName, my->field1, my->field2, ...)
+ * ```
+ *
+ * will generate comparison operators semantically equivalent to:
+ * ```
+ * bool operator<(const ClassName& other) {
+ *   return field1 < other.field1 && field2 < other.field2 && ...;
+ * }
+ * ```
+ */
+#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
+    bool operator COMPARATOR(const MY_TYPE& other) const { \
+      const MY_TYPE* me = this; \
+      auto fields1 = std::make_tuple( FIELDS ); \
+      me = &other; \
+      auto fields2 = std::make_tuple( FIELDS ); \
+      return fields1 COMPARATOR fields2; \
+    }
+#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
+#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_CMP(args...) \
+    GENERATE_EQUAL(args) \
+    GENERATE_LEQ(args)
+
 namespace nix {
 
 struct DrvOutput {
@@ -17,13 +45,7 @@ struct DrvOutput {
 
     static DrvOutput parse(const std::string &);
 
-    bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
-    bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
-
-private:
-    // Just to make comparison operators easier to write
-    std::pair<Hash, std::string> to_pair() const
-    { return std::make_pair(drvHash, outputName); }
+    GENERATE_CMP(DrvOutput, me->drvHash, me->outputName);
 };
 
 struct Realisation {
@@ -32,8 +54,60 @@ struct Realisation {
 
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
+
+    StorePath getPath() const { return outPath; }
+
+    GENERATE_CMP(Realisation, me->id, me->outPath);
 };
 
-typedef std::map<DrvOutput, Realisation> DrvOutputs;
+struct OpaquePath {
+    StorePath path;
+
+    StorePath getPath() const { return path; }
+
+    GENERATE_CMP(OpaquePath, me->path);
+};
+
+
+/**
+ * A store path with all the history of how it went into the store
+ */
+struct RealisedPath {
+    /*
+     * A path is either the result of the realisation of a derivation or
+     * an opaque blob that has been directly added to the store
+     */
+    using Raw = std::variant<Realisation, OpaquePath>;
+    Raw raw;
+
+    using Set = std::set<RealisedPath>;
+
+    RealisedPath(StorePath path) : raw(OpaquePath{path}) {}
+    RealisedPath(Realisation r) : raw(r) {}
+
+    /**
+     * Syntactic sugar to run `std::visit` on the raw value:
+     * path.visit(blah) == std::visit(blah, path.raw)
+     */
+    template <class Visitor>
+    constexpr decltype(auto) visit(Visitor && vis) {
+        return std::visit(vis, raw);
+    }
+    template <class Visitor>
+    constexpr decltype(auto) visit(Visitor && vis) const {
+        return std::visit(vis, raw);
+    }
+
+    /**
+     * Get the raw store path associated to this
+     */
+    StorePath path() const;
+
+    void closure(Store& store, Set& ret) const;
+    static void closure(Store& store, const Set& startPaths, Set& ret);
+    Set closure(Store& store) const;
+
+    GENERATE_CMP(RealisedPath, me->raw);
+};
 
 }
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index f15031a45..c56a1def1 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -16,6 +16,8 @@ struct CmdCopy : StorePathsCommand
 
     SubstituteFlag substitute = NoSubstitute;
 
+    using StorePathsCommand::run;
+
     CmdCopy()
         : StorePathsCommand(true)
     {

From 991edaace57d50d571f4f4658ef2d52b94a07f2c Mon Sep 17 00:00:00 2001
From: James Ottaway <jamesottaway@users.noreply.github.com>
Date: Fri, 29 Jan 2021 13:55:18 +1000
Subject: [PATCH 705/882] Shorten `mktemp` flag for macOS

Address `mktemp: illegal option -- -`.
---
 src/nix/develop.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 578258394..3c44fdb0e 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -239,7 +239,7 @@ struct Common : InstallableCommand, MixProfile
 
         out << buildEnvironment.bashFunctions << "\n";
 
-        out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
+        out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
 

From d5acc4865c8a5853bc5ede606d98c8055f8afdb2 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Fri, 29 Jan 2021 18:31:40 +0100
Subject: [PATCH 706/882] Use passthru for perl-bindings, allows Nix patching
 for Hydra

This allows patching Nix for Hydra with additional overlays, because
`.overrideAttrs` and co. will persist the passthru's
---
 flake.nix | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index 9addccd63..830cceb9f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -115,7 +115,7 @@
       # 'nix.perl-bindings' packages.
       overlay = final: prev: {
 
-        nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
+        nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
 
@@ -163,9 +163,8 @@
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           separateDebugInfo = true;
-        }) // {
 
-          perl-bindings = with final; stdenv.mkDerivation {
+          passthru.perl-bindings = with final; stdenv.mkDerivation {
             name = "nix-perl-${version}";
 
             src = self;

From d0b74e2d2506b9237263ad1294eb7297c99a5e1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 1 Feb 2021 13:11:42 +0000
Subject: [PATCH 707/882] --no-net -> --offline

---
 src/nix/main.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 58b643cc5..e95b04d85 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -91,7 +91,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         });
 
         addFlag({
-            .longName = "no-net",
+            .longName = "offline",
             .description = "Disable substituters and consider all previously downloaded files up-to-date.",
             .handler = {[&]() { useNet = false; }},
         });

From fb00e7dc529f54e6b2d864532e93ef3645b1b704 Mon Sep 17 00:00:00 2001
From: Dominik Schrempf <dominik.schrempf@gmail.com>
Date: Mon, 1 Feb 2021 17:42:14 +0100
Subject: [PATCH 708/882] Remove newline in operator table.

---
 doc/manual/src/expressions/language-operators.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md
index 1d787ffe3..b7fd6f4c6 100644
--- a/doc/manual/src/expressions/language-operators.md
+++ b/doc/manual/src/expressions/language-operators.md
@@ -25,5 +25,4 @@ order of precedence (from strongest to weakest binding).
 | Inequality               | *e1* `!=` *e2*                      | none          | Inequality.                                                                                                                                                                                                                   | 11         |
 | Logical AND              | *e1* `&&` *e2*                      | left          | Logical AND.                                                                                                                                                                                                                  | 12         |
 | Logical OR               | *e1* `\|\|` *e2*                    | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
-| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to `!e1 \|\|
-        e2`).                                                                                                                                                                    | 14         |
+| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to `!e1 \|\| e2`).                                                                                                                                                                            | 14         |

From 3d1bbabe55eff6e67d91e0cbee781c2b756a2e92 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 2 Feb 2021 19:50:03 -0600
Subject: [PATCH 709/882] Use derivation output name from toDerivation

This fixes an issue where derivations with a primary output that is
not "out" would fail with:

$ nix profile install nixpkgs#sqlite
error: opening directory '/nix/store/2a2ydlgyydly5czcc8lg12n6qqkfz863-sqlite-3.34.1-bin': No such file or directory

This happens because while derivations produce every output when
built, you might not have them if you didn't build the derivation
yourself (for instance, the store path was fetch from a binary cache).
This uses outputName provided from DerivationInfo which appears to
match the first output of the derivation.
---
 src/nix/profile.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 765d6866e..827f8be5a 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     attrPath,
                 };
 
-                pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
+                pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}});
 
                 manifest.elements.emplace_back(std::move(element));
             } else {

From 76d8bdfe355aa1976580f4fa8f11f1ec505a6c66 Mon Sep 17 00:00:00 2001
From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org>
Date: Tue, 2 Feb 2021 23:04:36 +0100
Subject: [PATCH 710/882] Include note about type of catched errors in tryEval
 documentation

Reference #356.
---
 src/libexpr/primops.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 13565b950..1d1afa768 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -696,10 +696,14 @@ static RegisterPrimOp primop_tryEval({
       Try to shallowly evaluate *e*. Return a set containing the
       attributes `success` (`true` if *e* evaluated successfully,
       `false` if an error was thrown) and `value`, equalling *e* if
-      successful and `false` otherwise. Note that this doesn't evaluate
-      *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval
-      e).success ` will be `true`. Using ` builtins.deepSeq ` one can
-      get the expected result: `let e = { x = throw ""; }; in
+      successful and `false` otherwise. `tryEval` will only prevent
+      errors created by `throw` or `assert` from being thrown.
+      Errors `tryEval` will not catch are for example those created
+      by `abort` and type errors generated by builtins. Also note that
+      this doesn't evaluate *e* deeply, so `let e = { x = throw ""; };
+      in (builtins.tryEval e).success` will be `true`. Using
+      `builtins.deepSeq` one can get the expected result:
+      `let e = { x = throw ""; }; in
       (builtins.tryEval (builtins.deepSeq e e)).success` will be
       `false`.
     )",

From e38cd5becbbff57951b6a576dd793f4777a9833c Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 3 Feb 2021 21:22:11 -0600
Subject: [PATCH 711/882] Always enter first level of attrset in nix search

This makes nix search always go through the first level of an
attribute set, even if it's not a top level attribute. For instance,
you can now list all GHC compilers with:

$ nix search nixpkgs#haskell.compiler
...

This is similar to how nix-env works when you pass in -A.
---
 src/nix/search.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 9f864b3a4..c52a48d4e 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -81,9 +81,9 @@ struct CmdSearch : InstallableCommand, MixJSON
 
         uint64_t results = 0;
 
-        std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)> visit;
+        std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
 
-        visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)
+        visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@@ -94,7 +94,7 @@ struct CmdSearch : InstallableCommand, MixJSON
                         auto cursor2 = cursor.getAttr(attr);
                         auto attrPath2(attrPath);
                         attrPath2.push_back(attr);
-                        visit(*cursor2, attrPath2);
+                        visit(*cursor2, attrPath2, false);
                     }
                 };
 
@@ -150,6 +150,9 @@ struct CmdSearch : InstallableCommand, MixJSON
                     || (attrPath[0] == "packages" && attrPath.size() <= 2))
                     recurse();
 
+                else if (initialRecurse)
+                    recurse();
+
                 else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
                     auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
                     if (attr && attr->getBool())
@@ -163,7 +166,7 @@ struct CmdSearch : InstallableCommand, MixJSON
         };
 
         for (auto & [cursor, prefix] : installable->getCursors(*state))
-            visit(*cursor, parseAttrPath(*state, prefix));
+            visit(*cursor, parseAttrPath(*state, prefix), true);
 
         if (!json && !results)
             throw Error("no results for the given search term(s)!");

From ca8facefb6b6b0ffd6e22507111847dbfc9a3c75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Thu, 4 Feb 2021 14:47:28 +0100
Subject: [PATCH 712/882] Normalize some error messages

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libcmd/installables.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 98a27ded9..9ad02b5f0 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -724,14 +724,13 @@ std::set<RealisedPath> toRealisedPaths(
                         if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
                             if (!outputHashes.count(output.first))
                                 throw Error(
-                                    "The derivation %s doesn't have an output "
-                                    "named %s",
+                                    "the derivation '%s' doesn't have an output named '%s'",
                                     store->printStorePath(bfd.drvPath),
                                     output.first);
                             auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
                             auto realisation = store->queryRealisation(outputId);
                             if (!realisation)
-                                throw Error("Cannot operate on output of unbuilt CA drv %s", outputId.to_string());
+                                throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
                             res.insert(RealisedPath{*realisation});
                         }
                         else {

From 43d409f6690b79b5d4e1ab5e9780de93eb0f677a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Thu, 4 Feb 2021 14:47:56 +0100
Subject: [PATCH 713/882] Fix a whitespace issue

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/realisation.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index c9b66186f..e4276c040 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -65,7 +65,7 @@ void RealisedPath::closure(
     ret.insert(pathsClosure.begin(), pathsClosure.end());
 }
 
-void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const
+void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const
 {
     RealisedPath::closure(store, {*this}, ret);
 }

From d2091af231ab97b729c2486b55e520c565e59dd3 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 4 Feb 2021 15:11:05 +0100
Subject: [PATCH 714/882] Move the GENERATE_CMP macro to its own file

Despite being an ugly hack, it can probably be useful in a couple extra
places
---
 src/libstore/realisation.hh | 29 +----------------------------
 src/libutil/comparator.hh   | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 28 deletions(-)
 create mode 100644 src/libutil/comparator.hh

diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 1ecddc4d1..557f54362 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -2,34 +2,7 @@
 
 #include "path.hh"
 #include <nlohmann/json_fwd.hpp>
-
-
-/* Awfull hacky generation of the comparison operators by doing a lexicographic
- * comparison between the choosen fields
- * ```
- * GENERATE_CMP(ClassName, my->field1, my->field2, ...)
- * ```
- *
- * will generate comparison operators semantically equivalent to:
- * ```
- * bool operator<(const ClassName& other) {
- *   return field1 < other.field1 && field2 < other.field2 && ...;
- * }
- * ```
- */
-#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
-    bool operator COMPARATOR(const MY_TYPE& other) const { \
-      const MY_TYPE* me = this; \
-      auto fields1 = std::make_tuple( FIELDS ); \
-      me = &other; \
-      auto fields2 = std::make_tuple( FIELDS ); \
-      return fields1 COMPARATOR fields2; \
-    }
-#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
-#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
-#define GENERATE_CMP(args...) \
-    GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args)
+#include "comparator.hh"
 
 namespace nix {
 
diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh
new file mode 100644
index 000000000..0315dc506
--- /dev/null
+++ b/src/libutil/comparator.hh
@@ -0,0 +1,30 @@
+#pragma once
+
+/* Awfull hacky generation of the comparison operators by doing a lexicographic
+ * comparison between the choosen fields.
+ *
+ * ```
+ * GENERATE_CMP(ClassName, me->field1, me->field2, ...)
+ * ```
+ *
+ * will generate comparison operators semantically equivalent to:
+ *
+ * ```
+ * bool operator<(const ClassName& other) {
+ *   return field1 < other.field1 && field2 < other.field2 && ...;
+ * }
+ * ```
+ */
+#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
+    bool operator COMPARATOR(const MY_TYPE& other) const { \
+      const MY_TYPE* me = this; \
+      auto fields1 = std::make_tuple( FIELDS ); \
+      me = &other; \
+      auto fields2 = std::make_tuple( FIELDS ); \
+      return fields1 COMPARATOR fields2; \
+    }
+#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
+#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_CMP(args...) \
+    GENERATE_EQUAL(args) \
+    GENERATE_LEQ(args)

From e69cfdebb090b3aabbff69a44504883d5b6fb866 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 4 Feb 2021 15:15:22 +0100
Subject: [PATCH 715/882] Remove the `visit` machinery in `RealisedPath`

In addition to being some ugly template trickery, it was also totally
useless as it was used in only one place where I could replace it by
just a few extra characters
---
 src/libstore/realisation.cc |  2 +-
 src/libstore/realisation.hh | 13 -------------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index e4276c040..cd74af4ee 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -47,7 +47,7 @@ Realisation Realisation::fromJSON(
 }
 
 StorePath RealisedPath::path() const {
-    return visit([](auto && arg) { return arg.getPath(); });
+    return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }
 
 void RealisedPath::closure(
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 557f54362..7c91d802a 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -58,19 +58,6 @@ struct RealisedPath {
     RealisedPath(StorePath path) : raw(OpaquePath{path}) {}
     RealisedPath(Realisation r) : raw(r) {}
 
-    /**
-     * Syntactic sugar to run `std::visit` on the raw value:
-     * path.visit(blah) == std::visit(blah, path.raw)
-     */
-    template <class Visitor>
-    constexpr decltype(auto) visit(Visitor && vis) {
-        return std::visit(vis, raw);
-    }
-    template <class Visitor>
-    constexpr decltype(auto) visit(Visitor && vis) const {
-        return std::visit(vis, raw);
-    }
-
     /**
      * Get the raw store path associated to this
      */

From 0187838e2e7ff01f1b480e3e85d9e96da0b4b78e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Feb 2021 12:11:50 +0100
Subject: [PATCH 716/882] Add a trace to readLine() failures

Hopefully this helps to diagnose 'error: unexpected EOF reading a
line' on macOS.
---
 src/libstore/build/derivation-goal.cc | 25 ++++++++++++++++++++++---
 tests/init.sh                         |  1 +
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 8717499c0..190adf31c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1044,7 +1044,14 @@ HookReply DerivationGoal::tryBuildHook()
            whether the hook wishes to perform the build. */
         string reply;
         while (true) {
-            string s = readLine(worker.hook->fromHook.readSide.get());
+            auto s = [&]() {
+                try {
+                    return readLine(worker.hook->fromHook.readSide.get());
+                } catch (Error & e) {
+                    e.addTrace({}, "while reading the response from the build hook");
+                    throw e;
+                }
+            }();
             if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
                 ;
             else if (string(s, 0, 2) == "# ") {
@@ -1084,7 +1091,12 @@ HookReply DerivationGoal::tryBuildHook()
 
     hook = std::move(worker.hook);
 
-    machineName = readLine(hook->fromHook.readSide.get());
+    try {
+        machineName = readLine(hook->fromHook.readSide.get());
+    } catch (Error & e) {
+        e.addTrace({}, "while reading the machine name from the build hook");
+        throw e;
+    }
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
@@ -1773,7 +1785,14 @@ void DerivationGoal::startBuilder()
 
     /* Check if setting up the build environment failed. */
     while (true) {
-        string msg = readLine(builderOut.readSide.get());
+        string msg = [&]() {
+            try {
+                return readLine(builderOut.readSide.get());
+            } catch (Error & e) {
+                e.addTrace({}, "while reading the response of setting up the build environment");
+                throw e;
+            }
+        }();
         if (string(msg, 0, 1) == "\2") break;
         if (string(msg, 0, 1) == "\1") {
             FdSource source(builderOut.readSide.get());
diff --git a/tests/init.sh b/tests/init.sh
index 63cf895e2..1a6ccb6fe 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -21,6 +21,7 @@ experimental-features = nix-command flakes
 gc-reserved-space = 0
 substituters =
 flake-registry = $TEST_ROOT/registry.json
+show-trace = true
 include nix.conf.extra
 EOF
 

From 480426a364f09e7992230b32f2941a09fb52d729 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Feb 2021 15:57:33 +0100
Subject: [PATCH 717/882] Add more instrumentation for #4270

---
 src/libstore/build/derivation-goal.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 190adf31c..eeaec4f2c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1784,12 +1784,14 @@ void DerivationGoal::startBuilder()
     worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
 
     /* Check if setting up the build environment failed. */
+    std::vector<std::string> msgs;
     while (true) {
         string msg = [&]() {
             try {
                 return readLine(builderOut.readSide.get());
             } catch (Error & e) {
-                e.addTrace({}, "while reading the response of setting up the build environment");
+                e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
+                    concatStringsSep("|", msgs));
                 throw e;
             }
         }();
@@ -1801,6 +1803,7 @@ void DerivationGoal::startBuilder()
             throw ex;
         }
         debug("sandbox setup: " + msg);
+        msgs.push_back(std::move(msg));
     }
 }
 

From d0e34c85f85510cb2ef591de29693b4cf8bdc65b Mon Sep 17 00:00:00 2001
From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org>
Date: Sat, 6 Feb 2021 12:59:11 +0100
Subject: [PATCH 718/882] libcmd/markdown: handle allocation errors in
 lowdown_term_rndr

We upgrade to lowdown 0.8.0 [1] which contains a fix/improvement to a
behavior mentioned in this issue thread [2] where a big part of
lowdown's API would just call exit(1) on allocation errors since that
is a satisfying behavior for the lowdown binary.

Now lowdown_term_rndr returns 0 if an allocation error occurred which we
check for in libcmd/markdown.cc.

Also the extern "C" { } wrapper around lowdown.h has been removed as it
is not necessary.

[1]: https://github.com/kristapsdz/lowdown/blob/6ca7c855a063d1c77ae0b89405047cc3913a74d8/versions.xml#L987-L1006
[2]: https://github.com/kristapsdz/lowdown/issues/45#issuecomment-756681153
---
 flake.nix              | 8 ++++----
 src/libcmd/markdown.cc | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/flake.nix b/flake.nix
index d94da9dae..8c60934e6 100644
--- a/flake.nix
+++ b/flake.nix
@@ -198,12 +198,12 @@
 
         };
 
-        lowdown = with final; stdenv.mkDerivation {
-          name = "lowdown-0.7.9";
+        lowdown = with final; stdenv.mkDerivation rec {
+          name = "lowdown-0.8.0";
 
           src = fetchurl {
-            url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz;
-            hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ==";
+            url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
+            hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
           };
 
           #src = lowdown-src;
diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc
index 40788a42f..d25113d93 100644
--- a/src/libcmd/markdown.cc
+++ b/src/libcmd/markdown.cc
@@ -3,9 +3,7 @@
 #include "finally.hh"
 
 #include <sys/queue.h>
-extern "C" {
 #include <lowdown.h>
-}
 
 namespace nix {
 
@@ -42,7 +40,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot allocate Markdown output buffer");
     Finally freeBuffer([&]() { lowdown_buf_free(buf); });
 
-    lowdown_term_rndr(buf, nullptr, renderer, node);
+    int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+    if (!rndr_res)
+        throw Error("allocation error while rendering Markdown");
 
     return std::string(buf->data, buf->size);
 }

From 6af26b7aec28e8bf1786ead3ba26beb50317c167 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Sat, 6 Feb 2021 13:29:38 +0100
Subject: [PATCH 719/882] Add Stale bot

The configuration was taken from nixpkgs repository and adjusted to
`NixOS/nix`.

A `stale` label was added to the labels (with gray color).

Issues and PRs with `critical` label are excluded from interacting with the
stale bot.
---
 .github/STALE-BOT.md | 35 +++++++++++++++++++++++++++++++++++
 .github/stale.yml    |  9 +++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 .github/STALE-BOT.md
 create mode 100644 .github/stale.yml

diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md
new file mode 100644
index 000000000..6cc03f540
--- /dev/null
+++ b/.github/STALE-BOT.md
@@ -0,0 +1,35 @@
+# Stale bot information
+
+- Thanks for your contribution!
+- To remove the stale label, just leave a new comment.
+- _How to find the right people to ping?_ &rarr; [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.)
+- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos).
+
+## Suggestions for PRs
+
+1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else.
+2. If it is unfinished but you plan to finish it, please mark it as a draft.
+3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work.
+4. To get things rolling again, rebase the PR against the target branch and address valid comments.
+5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604).
+6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions.
+
+## Suggestions for issues
+
+1. If it is resolved (either for you personally, or in general), please consider closing it.
+2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough.
+3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/).
+4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude!
+
+**Memorandum on closing issues**
+
+Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort.
+
+## Useful GitHub search queries
+
+- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nixs/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+)
+- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22)
+- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+)
+- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+)
+- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+)
+- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+)
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 000000000..f81b4c762
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,9 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+daysUntilStale: 180
+daysUntilClose: false
+exemptLabels:
+  - "critical"
+staleLabel: "2.status: stale"
+markComment: |
+  I marked this as stale due to inactivity. &rarr; [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)
+closeComment: false

From 91d83426f70bbf28c1bf92be5f662d76d1d47578 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Sat, 6 Feb 2021 13:33:34 +0100
Subject: [PATCH 720/882] typo

---
 .github/STALE-BOT.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md
index 6cc03f540..5e8f5d929 100644
--- a/.github/STALE-BOT.md
+++ b/.github/STALE-BOT.md
@@ -27,7 +27,7 @@ Don't be afraid to close an issue that holds valuable information. Closed issues
 
 ## Useful GitHub search queries
 
-- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nixs/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+)
+- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+)
 - [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22)
 - [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+)
 - [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+)

From 7c112351d9e941567e64063638396259546d9a48 Mon Sep 17 00:00:00 2001
From: Alyssa Ross <hi@alyssa.is>
Date: Sun, 7 Feb 2021 13:56:50 +0000
Subject: [PATCH 721/882] libutil: EPERM from kill(-1, ...) is fine

I tested a trivial program that called kill(-1, SIGKILL), which was
run as the only process for an unpriveleged user, on Linux and
FreeBSD.  On Linux, kill reported success, while on FreeBSD it failed
with EPERM.

POSIX says:

> If pid is -1, sig shall be sent to all processes (excluding an
> unspecified set of system processes) for which the process has
> permission to send that signal.

and

> The kill() function is successful if the process has permission to
> send sig to any of the processes specified by pid.  If kill() fails,
> no signal shall be sent.

and

> [EPERM]
>     The process does not have permission to send the signal to any
>     receiving process.

My reading of this is that kill(-1, ...) may fail with EPERM when
there are no other processes to kill (since the current process is
ignored).  Since kill(-1, ...) only attempts to kill processes the
user has permission to kill, it can't mean that we tried to do
something we didn't have permission to kill, so it should be fine to
interpret EPERM the same as success here for any POSIX-compliant
system.

This fixes an issue that Mic92 encountered[1] when he tried to review a
Nixpkgs PR on FreeBSD.

[1]: https://github.com/NixOS/nixpkgs/pull/81459#issuecomment-606073668
---
 src/libutil/util.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 89f7b58f8..ef37275ac 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -946,7 +946,7 @@ void killUser(uid_t uid)
 #else
             if (kill(-1, SIGKILL) == 0) break;
 #endif
-            if (errno == ESRCH) break; /* no more processes */
+            if (errno == ESRCH || errno == EPERM) break; /* no more processes */
             if (errno != EINTR)
                 throw SysError("cannot kill processes for uid '%1%'", uid);
         }

From 37352aa7e19e0bfebbd0c32985cbf79a83508538 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 7 Feb 2021 20:44:56 +0100
Subject: [PATCH 722/882] Support --no-net for backwards compatibility

---
 src/libutil/args.cc | 3 +++
 src/libutil/args.hh | 1 +
 src/nix/main.cc     | 1 +
 3 files changed, 5 insertions(+)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 71bae0504..9377fe4c0 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -14,6 +14,8 @@ void Args::addFlag(Flag && flag_)
         assert(flag->handler.arity == flag->labels.size());
     assert(flag->longName != "");
     longFlags[flag->longName] = flag;
+    for (auto & alias : flag->aliases)
+        longFlags[alias] = flag;
     if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
@@ -191,6 +193,7 @@ nlohmann::json Args::toJSON()
 
     for (auto & [name, flag] : longFlags) {
         auto j = nlohmann::json::object();
+        if (flag->aliases.count(name)) continue;
         if (flag->shortName)
             j["shortName"] = std::string(1, flag->shortName);
         if (flag->description != "")
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 42d8515ef..88f068087 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -97,6 +97,7 @@ protected:
         typedef std::shared_ptr<Flag> ptr;
 
         std::string longName;
+        std::set<std::string> aliases;
         char shortName = 0;
         std::string description;
         std::string category;
diff --git a/src/nix/main.cc b/src/nix/main.cc
index e95b04d85..ef5e41a55 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -92,6 +92,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
         addFlag({
             .longName = "offline",
+            .aliases = {"no-net"}, // FIXME: remove
             .description = "Disable substituters and consider all previously downloaded files up-to-date.",
             .handler = {[&]() { useNet = false; }},
         });

From bab3f30755490207446966e9e828119462b57141 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Mon, 8 Feb 2021 11:49:07 +0100
Subject: [PATCH 723/882] Auto closing issues/PRs after 1year.

---
 .github/stale.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/stale.yml b/.github/stale.yml
index f81b4c762..fe24942f4 100644
--- a/.github/stale.yml
+++ b/.github/stale.yml
@@ -1,9 +1,10 @@
 # Configuration for probot-stale - https://github.com/probot/stale
 daysUntilStale: 180
-daysUntilClose: false
+daysUntilClose: 365
 exemptLabels:
   - "critical"
-staleLabel: "2.status: stale"
+staleLabel: "stale"
 markComment: |
   I marked this as stale due to inactivity. &rarr; [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)
-closeComment: false
+closeComment: |
+  I closed this issue due to inactivity. &rarr; [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)

From f2245091d033a8037aeb29ae701d20611500af6d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 9 Feb 2021 12:26:41 -0500
Subject: [PATCH 724/882] Revert "narinfo: Change NAR URLs to be addressed on
 the NAR hash instead of the compressed hash"

---
 src/libstore/binary-cache-store.cc | 6 +++++-
 tests/binary-cache.sh              | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 15163ead5..4f5f8607d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -176,7 +176,11 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;
     narInfo->fileSize = fileSize;
-    narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar";
+    narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index f8d47170f..6697ce236 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -60,7 +60,7 @@ basicDownloadTests
 # Test whether Nix notices if the NAR doesn't match the hash in the NAR info.
 clearStore
 
-nar=$(ls $cacheDir/nar/*.nar | head -n1)
+nar=$(ls $cacheDir/nar/*.nar.xz | head -n1)
 mv $nar $nar.good
 mkdir -p $TEST_ROOT/empty
 nix-store --dump $TEST_ROOT/empty | xz > $nar

From ad337c8697099ac9deb6e0ac16ea91d8acc51e4f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 12 Feb 2021 17:33:28 +0000
Subject: [PATCH 725/882] Deeper `Command` hierarchy to remove redundancy

Simply put, we now have `StorePathCommand : public StorePathsCommand` so
`StorePathCommand` doesn't reimplement work.
---
 src/libcmd/command.cc | 4 +---
 src/libcmd/command.hh | 6 +++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index efdc98d5a..d29954f67 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -118,10 +118,8 @@ void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
     run(store, std::move(storePaths));
 }
 
-void StorePathCommand::run(ref<Store> store)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
 {
-    auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
-
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
 
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 8c0b3a94a..c02193924 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -177,13 +177,13 @@ struct StorePathsCommand : public RealisedPathsCommand
 };
 
 /* A command that operates on exactly one store path. */
-struct StorePathCommand : public InstallablesCommand
+struct StorePathCommand : public StorePathsCommand
 {
-    using StoreCommand::run;
+    using StorePathsCommand::run;
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store) override;
+    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
 };
 
 /* A helper class for registering commands globally. */

From 35129884f9348f068d538e67bb559cc6104f714e Mon Sep 17 00:00:00 2001
From: Mauricio Scheffer <mauricioscheffer@gmail.com>
Date: Tue, 16 Feb 2021 23:19:42 +0000
Subject: [PATCH 726/882] Fix Haskell example

http://nixos.org redirects to https://nixos.org and apparently the HTTP library doesn't follow the redirect, so the output is empty.
When defining https in the request it crashes because the library doesn't seem to support https.
So this switches the example to a different http library.
---
 doc/manual/src/command-ref/nix-shell.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 88b675e71..938d56e6e 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -232,22 +232,23 @@ terraform apply
 > in a nix-shell shebang.
 
 Finally, using the merging of multiple nix-shell shebangs the following
-Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable
+Haskell script uses a specific branch of Nixpkgs/NixOS (the 20.03 stable
 branch):
 
 ```haskell
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz
 
-import Network.HTTP
+import Network.Curl.Download
 import Text.HTML.TagSoup
+import Data.Either
+import Data.ByteString.Char8 (unpack)
 
 -- Fetch nixos.org and print all hrefs.
 main = do
-  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
-  body <- getResponseBody resp
-  let tags = filter (isTagOpenName "a") $ parseTags body
+  resp <- openURI "https://nixos.org/"
+  let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp
   let tags' = map (fromAttrib "href") tags
   mapM_ putStrLn $ filter (/= "") tags'
 ```

From 5f4701e70d35bb9ea2fb659caf387a30001e28ce Mon Sep 17 00:00:00 2001
From: Mauricio Scheffer <mauricioscheffer@gmail.com>
Date: Tue, 16 Feb 2021 23:27:04 +0000
Subject: [PATCH 727/882] Update doc/manual/src/command-ref/nix-shell.md

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 doc/manual/src/command-ref/nix-shell.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 938d56e6e..54812a49f 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -238,7 +238,7 @@ branch):
 ```haskell
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
 
 import Network.Curl.Download
 import Text.HTML.TagSoup

From 6042febfce3011aaa5e3c369ea14a0d93ad2880e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 15:30:49 +0100
Subject: [PATCH 728/882] Restore warning about 'nix' being experimental

Fixes #4552.
---
 doc/manual/generate-manpage.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index a563c31f8..964b57086 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -7,7 +7,10 @@ let
 
   showCommand =
     { command, def, filename }:
-    "# Name\n\n"
+    ''
+      **Warning**: This program is **experimental** and its interface is subject to change.
+    ''
+    + "# Name\n\n"
     + "`${command}` - ${def.description}\n\n"
     + "# Synopsis\n\n"
     + showSynopsis { inherit command; args = def.args; }

From 063de66909ff1b20394cdebdca1ef62bb6ca1d51 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 16:42:03 +0100
Subject: [PATCH 729/882] nix develop: Fix quoted string handling

Fixes #4540.
---
 src/nix/develop.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 3c44fdb0e..0938cbe5b 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -59,7 +59,7 @@ BuildEnvironment readEnvironment(const Path & path)
         R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
 
     static std::string squotedStringRegex =
-        R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
+        R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
 
     static std::string indexedArrayRegex =
         R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";

From cced73496b835b545be91cbebc4f89f61a7b106f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 16:53:19 +0100
Subject: [PATCH 730/882] nix flake show: Handle 'overlays' output

Fixes #4542.
---
 src/nix/flake.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 4cd7d77a0..091af8084 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -880,7 +880,8 @@ struct CmdFlakeShow : FlakeCommand
                             || attrPath[0] == "nixosConfigurations"
                             || attrPath[0] == "nixosModules"
                             || attrPath[0] == "defaultApp"
-                            || attrPath[0] == "templates"))
+                            || attrPath[0] == "templates"
+                            || attrPath[0] == "overlays"))
                     || ((attrPath.size() == 1 || attrPath.size() == 2)
                         && (attrPath[0] == "checks"
                             || attrPath[0] == "packages"
@@ -943,7 +944,8 @@ struct CmdFlakeShow : FlakeCommand
                 else {
                     logger->cout("%s: %s",
                         headerPrefix,
-                        attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" :
+                        (attrPath.size() == 1 && attrPath[0] == "overlay")
+                        || (attrPath.size() == 2 && attrPath[0] == "overlays") ? "Nixpkgs overlay" :
                         attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" :
                         attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" :
                         ANSI_YELLOW "unknown" ANSI_NORMAL);

From f33878b6562c746d5865a86e64f02c75feaf5b3e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 17:11:14 +0100
Subject: [PATCH 731/882] Make 'nix --version -vv' work

Fixes #3743.
---
 src/nix/main.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index ef5e41a55..5f4eb8918 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -61,6 +61,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     bool printBuildLogs = false;
     bool useNet = true;
     bool refresh = false;
+    bool showVersion = false;
 
     NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
     {
@@ -87,7 +88,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         addFlag({
             .longName = "version",
             .description = "Show version information.",
-            .handler = {[&]() { if (!completions) printVersion(programName); }},
+            .handler = {[&]() { showVersion = true; }},
         });
 
         addFlag({
@@ -280,6 +281,11 @@ void mainWrapped(int argc, char * * argv)
 
     initPlugins();
 
+    if (args.showVersion) {
+        printVersion(programName);
+        return;
+    }
+
     if (!args.command)
         throw UsageError("no subcommand specified");
 

From 13897afbe6cf7ef8013c0c94109696bb7b13d0c0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 17:32:10 +0100
Subject: [PATCH 732/882] Throw an error if --arg / --argstr is used with a
 flake

Fixes #3949.
---
 src/libcmd/installables.cc | 24 ++++++++++++++++++++++--
 src/libcmd/installables.hh | 12 +++++++-----
 src/nix/bundle.cc          |  2 +-
 src/nix/develop.cc         |  1 +
 src/nix/flake.cc           |  2 +-
 src/nix/profile.cc         |  8 +++++++-
 6 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 9ad02b5f0..4739dc974 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -496,6 +496,23 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
     return s;
 }
 
+InstallableFlake::InstallableFlake(
+    SourceExprCommand * cmd,
+    ref<EvalState> state,
+    FlakeRef && flakeRef,
+    Strings && attrPaths,
+    Strings && prefixes,
+    const flake::LockFlags & lockFlags)
+    : InstallableValue(state),
+      flakeRef(flakeRef),
+      attrPaths(attrPaths),
+      prefixes(prefixes),
+      lockFlags(lockFlags)
+{
+    if (cmd && cmd->getAutoArgs(*state)->size())
+        throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
+}
+
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
     auto lockedFlake = getLockedFlake();
@@ -628,9 +645,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
-                        getEvalState(), std::move(flakeRef),
+                        this,
+                        getEvalState(),
+                        std::move(flakeRef),
                         fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
-                        getDefaultFlakeAttrPathPrefixes(), lockFlags));
+                        getDefaultFlakeAttrPathPrefixes(),
+                        lockFlags));
                 continue;
             } catch (...) {
                 ex = std::current_exception();
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index f37b3f829..b714f097b 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -104,11 +104,13 @@ struct InstallableFlake : InstallableValue
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
-    InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef,
-        Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags)
-        : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths),
-          prefixes(prefixes), lockFlags(lockFlags)
-    { }
+    InstallableFlake(
+        SourceExprCommand * cmd,
+        ref<EvalState> state,
+        FlakeRef && flakeRef,
+        Strings && attrPaths,
+        Strings && prefixes,
+        const flake::LockFlags & lockFlags);
 
     std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 1789e4598..48f4eb6e3 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -74,7 +74,7 @@ struct CmdBundle : InstallableCommand
 
         auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
-        auto bundler = InstallableFlake(
+        auto bundler = InstallableFlake(this,
             evalState, std::move(bundlerFlakeRef),
             Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
             Strings({"bundlers."}), lockFlags);
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 0938cbe5b..d0b140570 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -443,6 +443,7 @@ struct CmdDevelop : Common, MixEnvironment
             auto state = getEvalState();
 
             auto bashInstallable = std::make_shared<InstallableFlake>(
+                this,
                 state,
                 installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 091af8084..b9cde5d6d 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -595,7 +595,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
 
         auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
 
-        auto installable = InstallableFlake(
+        auto installable = InstallableFlake(nullptr,
             evalState, std::move(templateFlakeRef),
             Strings{templateName == "" ? "defaultTemplate" : templateName},
             Strings(attrsPathPrefixes), lockFlags);
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 827f8be5a..4d275f577 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -399,7 +399,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 Activity act(*logger, lvlChatty, actUnknown,
                     fmt("checking '%s' for updates", element.source->attrPath));
 
-                InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags);
+                InstallableFlake installable(
+                    this,
+                    getEvalState(),
+                    FlakeRef(element.source->originalRef),
+                    {element.source->attrPath},
+                    {},
+                    lockFlags);
 
                 auto [attrPath, resolvedRef, drv] = installable.toDerivation();
 

From 7bd9898d5ca72ed136032590745c56826317a328 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 17:54:13 +0100
Subject: [PATCH 733/882] nix run: Allow program name to be set in
 meta.mainProgram

This is useful when the program name doesn't match the package name
(e.g. ripgrep vs rg).

Fixes #4498.
---
 src/nix/app.cc | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/nix/app.cc b/src/nix/app.cc
index 80acbf658..cf147c631 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -12,11 +12,16 @@ App Installable::toApp(EvalState & state)
 
     auto type = cursor->getAttr("type")->getString();
 
+    auto checkProgram = [&](const Path & program)
+    {
+        if (!state.store->isInStore(program))
+            throw Error("app program '%s' is not in the Nix store", program);
+    };
+
     if (type == "app") {
         auto [program, context] = cursor->getAttr("program")->getStringWithContext();
 
-        if (!state.store->isInStore(program))
-            throw Error("app program '%s' is not in the Nix store", program);
+        checkProgram(program);
 
         std::vector<StorePathWithOutputs> context2;
         for (auto & [path, name] : context)
@@ -33,9 +38,17 @@ App Installable::toApp(EvalState & state)
         auto outPath = cursor->getAttr(state.sOutPath)->getString();
         auto outputName = cursor->getAttr(state.sOutputName)->getString();
         auto name = cursor->getAttr(state.sName)->getString();
+        auto aMeta = cursor->maybeGetAttr("meta");
+        auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
+        auto mainProgram =
+            aMainProgram
+            ? aMainProgram->getString()
+            : DrvName(name).name;
+        auto program = outPath + "/bin/" + mainProgram;
+        checkProgram(program);
         return App {
             .context = { { drvPath, {outputName} } },
-            .program = outPath + "/bin/" + DrvName(name).name,
+            .program = program,
         };
     }
 

From 1b578255245e2e1347059ad7d9171cf822c723a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Feb 2021 17:58:40 +0100
Subject: [PATCH 734/882] Document meta.mainProgram

Issue #4498.
---
 src/nix/run.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/nix/run.md b/src/nix/run.md
index c178e8b13..a76750376 100644
--- a/src/nix/run.md
+++ b/src/nix/run.md
@@ -43,9 +43,10 @@ program specified by the app definition.
 
 If *installable* evaluates to a derivation, it will try to execute the
 program `<out>/bin/<name>`, where *out* is the primary output store
-path of the derivation and *name* is the name part of the value of the
-`name` attribute of the derivation (e.g. if `name` is set to
-`hello-1.10`, it will run `$out/bin/hello`).
+path of the derivation and *name* is the `meta.mainProgram` attribute
+of the derivation if it exists, and otherwise the name part of the
+value of the `name` attribute of the derivation (e.g. if `name` is set
+to `hello-1.10`, it will run `$out/bin/hello`).
 
 # Flake output attributes
 

From cd44c0af71ace2eb8056c2b26b9249a5aa102b41 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 18 Feb 2021 19:22:37 +0100
Subject: [PATCH 735/882] Increase default stack size on Linux

Workaround for #4550.
---
 src/nix/main.cc | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 5f4eb8918..1b68cf15b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -17,6 +17,10 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
+#if __linux__
+#include <sys/resource.h>
+#endif
+
 #include <nlohmann/json.hpp>
 
 extern std::string chrootHelperName;
@@ -325,6 +329,17 @@ void mainWrapped(int argc, char * * argv)
 
 int main(int argc, char * * argv)
 {
+    // Increase the default stack size for the evaluator and for
+    // libstdc++'s std::regex.
+    #if __linux__
+    rlim_t stackSize = 64 * 1024 * 1024;
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+        limit.rlim_cur = stackSize;
+        setrlimit(RLIMIT_STACK, &limit);
+    }
+    #endif
+
     return nix::handleExceptions(argv[0], [&]() {
         nix::mainWrapped(argc, argv);
     });

From 263f6dbd1cef6eb9560737f6daf963f8968a65d8 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 8 Dec 2020 20:38:37 +0100
Subject: [PATCH 736/882] Don't crash nix-build when not all outputs are
 realised

Change the `nix-build` logic for linking/printing the output paths to allow for
some outputs to be missing. This might happen when the toplevel
derivation didn't have to be built, either because all the required
outputs were already there, or because they have all been substituted.
---
 src/nix-build/nix-build.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 361f9730d..d975cd16d 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -518,9 +518,11 @@ static void main_nix_build(int argc, char * * argv)
             if (counter)
                 drvPrefix += fmt("-%d", counter + 1);
 
-            auto builtOutputs = store->queryDerivationOutputMap(drvPath);
+            auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath);
 
-            auto outputPath = builtOutputs.at(outputName);
+            auto maybeOutputPath = builtOutputs.at(outputName);
+            assert(maybeOutputPath);
+            auto outputPath = *maybeOutputPath;
 
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
                 std::string symlink = drvPrefix;

From be1b5c4e59ca1c3504a44e2058807f7207432846 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 8 Dec 2020 18:11:33 +0100
Subject: [PATCH 737/882] Test the garbage collection of CA derivations

Simple test to ensure that `nix-build && nix-collect-garbage &&
nix-build -j0` works as it should
---
 tests/content-addressed.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh
index e8ac88609..7e32e1f28 100644
--- a/tests/content-addressed.sh
+++ b/tests/content-addressed.sh
@@ -48,6 +48,10 @@ testCutoff () {
 testGC () {
     nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
     nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
+    clearStore
+    buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA
+    nix-collect-garbage --experimental-features ca-derivations
+    buildAttr rootCA 1 -j0
 }
 
 testNixCommand () {

From 87c8d3d702123528ac068bb703232e24431c535e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 27 Jan 2021 10:03:05 +0100
Subject: [PATCH 738/882] Register the realisations for unresolved drvs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Once a build is done, get back to the original derivation, and register
all the newly built outputs for this derivation.

This allows Nix to work properly with derivations that don't have all
their build inputs available − thus allowing garbage collection and
(once it's implemented) binary substitution
---
 src/libstore/build/derivation-goal.cc | 54 ++++++++++++++++++++++++++-
 src/libstore/build/derivation-goal.hh |  3 ++
 src/libstore/derivations.cc           |  9 ++++-
 src/libstore/local-store.cc           |  2 +-
 src/libstore/local-store.hh           |  2 +-
 src/libstore/store-api.cc             | 15 +-------
 src/libstore/store-api.hh             |  6 ---
 7 files changed, 67 insertions(+), 24 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index eeaec4f2c..315cf3f0a 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -506,6 +506,7 @@ void DerivationGoal::inputsRealised()
             Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
+            resolvedDrv = drvResolved;
 
             auto msg = fmt("Resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
@@ -1019,7 +1020,45 @@ void DerivationGoal::buildDone()
 }
 
 void DerivationGoal::resolvedFinished() {
-    done(BuildResult::Built);
+    assert(resolvedDrv);
+
+    // If the derivation was originally a full `Derivation` (and not just
+    // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes`
+    // will be wrong otherwise
+    Derivation fullDrv = *drv;
+    if (auto upcasted = dynamic_cast<Derivation *>(drv.get()))
+        fullDrv = *upcasted;
+
+    auto originalHashes = staticOutputHashes(worker.store, fullDrv);
+    auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
+
+    // `wantedOutputs` might be empty, which means “all the outputs”
+    auto realWantedOutputs = wantedOutputs;
+    if (realWantedOutputs.empty())
+        realWantedOutputs = resolvedDrv->outputNames();
+
+    for (auto & wantedOutput : realWantedOutputs) {
+        assert(originalHashes.count(wantedOutput) != 0);
+        assert(resolvedHashes.count(wantedOutput) != 0);
+        auto realisation = worker.store.queryRealisation(
+                DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
+        );
+        // We've just built it, but maybe the build failed, in which case the
+        // realisation won't be there
+        if (realisation) {
+            auto newRealisation = *realisation;
+            newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput};
+            worker.store.registerDrvOutput(newRealisation);
+        } else {
+            // If we don't have a realisation, then it must mean that something
+            // failed when building the resolved drv
+            assert(!result.success());
+        }
+    }
+
+    // This is potentially a bit fishy in terms of error reporting. Not sure
+    // how to do it in a cleaner way
+    amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
 }
 
 HookReply DerivationGoal::tryBuildHook()
@@ -3804,6 +3843,19 @@ void DerivationGoal::checkPathValidity()
                     : PathStatus::Corrupt,
             };
         }
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            Derivation fullDrv = *drv;
+            if (auto upcasted = dynamic_cast<Derivation *>(drv.get()))
+                fullDrv = *upcasted;
+            auto outputHashes = staticOutputHashes(worker.store, fullDrv);
+            if (auto real = worker.store.queryRealisation(
+                    DrvOutput{outputHashes.at(i.first), i.first})) {
+                info.known = {
+                    .path = real->outPath,
+                    .status = PathStatus::Valid,
+                };
+            }
+        }
         initialOutputs.insert_or_assign(i.first, info);
     }
 }
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 8ee0be9e1..b7b85c85d 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -48,6 +48,9 @@ struct DerivationGoal : public Goal
     /* The path of the derivation. */
     StorePath drvPath;
 
+    /* The path of the corresponding resolved derivation */
+    std::optional<BasicDerivation> resolvedDrv;
+
     /* The specific outputs that we need to build.  Empty means all of
        them. */
     StringSet wantedOutputs;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 7466c7d41..4b774c42a 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -756,8 +756,13 @@ std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
         StringSet newOutputNames;
         for (auto & outputName : input.second) {
             auto actualPathOpt = inputDrvOutputs.at(outputName);
-            if (!actualPathOpt)
+            if (!actualPathOpt) {
+                warn("Input %s!%s missing, aborting the resolving",
+                    store.printStorePath(input.first),
+                    outputName
+                );
                 return std::nullopt;
+            }
             auto actualPath = *actualPathOpt;
             inputRewrites.emplace(
                 downstreamPlaceholder(store, input.first, outputName),
@@ -782,6 +787,8 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StoreP
     // This is quite dirty and leaky, but will disappear once #4340 is merged
     static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
 
+    debug("Trying to resolve %s", store.printStorePath(drvPath));
+
     {
         auto resolutions = resolutionsCache.lock();
         auto resolvedDrvIter = resolutions->find(drvPath);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index f45af2bac..e06c47cde 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 
 
 std::map<std::string, std::optional<StorePath>>
-LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
+LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
 {
     auto path = path_;
     auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 9d235ba0a..780cc0f07 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -127,7 +127,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 37c11fe86..2658f7617 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -366,7 +366,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
-std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapNoResolve(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
 {
     std::map<std::string, std::optional<StorePath>> outputs;
     auto drv = readInvalidDerivation(path);
@@ -376,19 +376,6 @@ std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapN
     return outputs;
 }
 
-std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
-{
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-        auto resolvedDrv = Derivation::tryResolve(*this, path);
-        if (resolvedDrv) {
-            auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true);
-            if (isValidPath(resolvedDrvPath))
-                return queryDerivationOutputMapNoResolve(resolvedDrvPath);
-        }
-    }
-    return queryDerivationOutputMapNoResolve(path);
-}
-
 OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
     auto resp = queryPartialDerivationOutputMap(path);
     OutputPathMap result;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9e98eb8f9..6dcd43ed1 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -415,12 +415,6 @@ public:
        `std::nullopt`.  */
     virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
 
-    /*
-     * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve
-     * the derivation
-     */
-    virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path);
-
     /* Query the mapping outputName=>outputPath for the given derivation.
        Assume every output has a mapping and throw an exception otherwise. */
     OutputPathMap queryDerivationOutputMap(const StorePath & path);

From 93d9eb78a0733c5adcbc6ee7b8a257605ae4a32f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Thu, 4 Feb 2021 11:12:24 +0100
Subject: [PATCH 739/882] Syntactic fixes

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/derivations.cc | 2 +-
 src/libstore/local-store.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 4b774c42a..36993ffc2 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -787,7 +787,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StoreP
     // This is quite dirty and leaky, but will disappear once #4340 is merged
     static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
 
-    debug("Trying to resolve %s", store.printStorePath(drvPath));
+    debug("trying to resolve %s", store.printStorePath(drvPath));
 
     {
         auto resolutions = resolutionsCache.lock();
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e06c47cde..0962418dd 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 
 
 std::map<std::string, std::optional<StorePath>>
-LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
+LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
 {
     auto path = path_;
     auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {

From 0bfbd043699908bcaff1493c733ab4798b642b82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Thu, 4 Feb 2021 11:13:38 +0100
Subject: [PATCH 740/882] Don't expose the "bang" drvoutput syntax

It's not fixed nor useful atm, so better keep it hidden

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/derivations.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 36993ffc2..7807089ca 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -757,9 +757,9 @@ std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
         for (auto & outputName : input.second) {
             auto actualPathOpt = inputDrvOutputs.at(outputName);
             if (!actualPathOpt) {
-                warn("Input %s!%s missing, aborting the resolving",
-                    store.printStorePath(input.first),
-                    outputName
+                warn("output %s of input %s missing, aborting the resolving",
+                    outputName,
+                    store.printStorePath(input.first)
                 );
                 return std::nullopt;
             }

From 4bc28c44f258f4f8c8a3935d1acf746f6abe3d8f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 4 Feb 2021 14:41:49 +0100
Subject: [PATCH 741/882] Store the output hashes in the initialOutputs of the
 drv goal

That way we
1. Don't have to recompute them several times
2. Can compute them in a place where we know the type of the parent
  derivation, meaning that we don't need the casting dance we had before
---
 src/libstore/build/derivation-goal.cc | 49 ++++++++++++++++-----------
 src/libstore/build/derivation-goal.hh |  1 +
 2 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 315cf3f0a..d8a89a2d0 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -124,6 +124,17 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
     , buildMode(buildMode)
 {
     this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
+
+    auto outputHashes = staticOutputHashes(worker.store, drv);
+    for (auto &[outputName, outputHash] : outputHashes)
+      initialOutputs.insert({
+            outputName,
+            InitialOutput{
+                .wanted = true, // Will be refined later
+                .outputHash = outputHash
+            }
+          });
+
     state = &DerivationGoal::haveDerivation;
     name = fmt(
         "building of '%s' from in-memory derivation",
@@ -258,8 +269,20 @@ void DerivationGoal::loadDerivation()
 
     assert(worker.store.isValidPath(drvPath));
 
+    auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath));
+
+    auto outputHashes = staticOutputHashes(worker.store, *fullDrv);
+    for (auto &[outputName, outputHash] : outputHashes)
+      initialOutputs.insert({
+            outputName,
+            InitialOutput{
+                .wanted = true, // Will be refined later
+                .outputHash = outputHash
+            }
+          });
+
     /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
+    drv = std::unique_ptr<BasicDerivation>(fullDrv);
 
     haveDerivation();
 }
@@ -1022,14 +1045,6 @@ void DerivationGoal::buildDone()
 void DerivationGoal::resolvedFinished() {
     assert(resolvedDrv);
 
-    // If the derivation was originally a full `Derivation` (and not just
-    // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes`
-    // will be wrong otherwise
-    Derivation fullDrv = *drv;
-    if (auto upcasted = dynamic_cast<Derivation *>(drv.get()))
-        fullDrv = *upcasted;
-
-    auto originalHashes = staticOutputHashes(worker.store, fullDrv);
     auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
 
     // `wantedOutputs` might be empty, which means “all the outputs”
@@ -1038,7 +1053,7 @@ void DerivationGoal::resolvedFinished() {
         realWantedOutputs = resolvedDrv->outputNames();
 
     for (auto & wantedOutput : realWantedOutputs) {
-        assert(originalHashes.count(wantedOutput) != 0);
+        assert(initialOutputs.count(wantedOutput) != 0);
         assert(resolvedHashes.count(wantedOutput) != 0);
         auto realisation = worker.store.queryRealisation(
                 DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
@@ -1047,7 +1062,7 @@ void DerivationGoal::resolvedFinished() {
         // realisation won't be there
         if (realisation) {
             auto newRealisation = *realisation;
-            newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput};
+            newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             worker.store.registerDrvOutput(newRealisation);
         } else {
             // If we don't have a realisation, then it must mean that something
@@ -3829,9 +3844,8 @@ void DerivationGoal::checkPathValidity()
 {
     bool checkHash = buildMode == bmRepair;
     for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput info {
-            .wanted = wantOutput(i.first, wantedOutputs),
-        };
+        InitialOutput & info = initialOutputs.at(i.first);
+        info.wanted = wantOutput(i.first, wantedOutputs);
         if (i.second) {
             auto outputPath = *i.second;
             info.known = {
@@ -3844,19 +3858,14 @@ void DerivationGoal::checkPathValidity()
             };
         }
         if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-            Derivation fullDrv = *drv;
-            if (auto upcasted = dynamic_cast<Derivation *>(drv.get()))
-                fullDrv = *upcasted;
-            auto outputHashes = staticOutputHashes(worker.store, fullDrv);
             if (auto real = worker.store.queryRealisation(
-                    DrvOutput{outputHashes.at(i.first), i.first})) {
+                    DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) {
                 info.known = {
                     .path = real->outPath,
                     .status = PathStatus::Valid,
                 };
             }
         }
-        initialOutputs.insert_or_assign(i.first, info);
     }
 }
 
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index b7b85c85d..761100d3a 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -37,6 +37,7 @@ struct InitialOutputStatus {
 
 struct InitialOutput {
     bool wanted;
+    Hash outputHash;
     std::optional<InitialOutputStatus> known;
 };
 

From f483b623e98a0feb2568e5be076b533c5838ba32 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 16 Feb 2021 08:16:12 +0100
Subject: [PATCH 742/882] Remove the drv resolution caching mechanism

It isn't needed anymore now that don't need to eagerly resolve
everything like we used to do. So we can safely get rid of it
---
 src/libstore/derivations.cc | 34 +---------------------------------
 src/libstore/derivations.hh |  4 ----
 2 files changed, 1 insertion(+), 37 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 7807089ca..6d0742b4f 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
     BasicDerivation resolved { *this };
 
     // Input paths that we'll want to rewrite in the derivation
@@ -776,36 +776,4 @@ std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
     return resolved;
 }
 
-std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
-{
-    auto drvPath = writeDerivation(store, *this, NoRepair, false);
-    return Derivation::tryResolve(store, drvPath);
-}
-
-std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
-{
-    // This is quite dirty and leaky, but will disappear once #4340 is merged
-    static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
-
-    debug("trying to resolve %s", store.printStorePath(drvPath));
-
-    {
-        auto resolutions = resolutionsCache.lock();
-        auto resolvedDrvIter = resolutions->find(drvPath);
-        if (resolvedDrvIter != resolutions->end()) {
-            auto & [_, resolvedDrv] = *resolvedDrvIter;
-                return *resolvedDrv;
-        }
-    }
-
-    /* Try resolve drv and use that path instead. */
-    auto drv = store.readDerivation(drvPath);
-    auto attempt = drv.tryResolveUncached(store);
-    if (!attempt)
-        return std::nullopt;
-    /* Store in memo table. */
-    resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
-    return *attempt;
-}
-
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 3d8f19aef..4e5985fab 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -138,14 +138,10 @@ struct Derivation : BasicDerivation
 
        2. Input placeholders are replaced with realized input store paths. */
     std::optional<BasicDerivation> tryResolve(Store & store);
-    static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
 
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
-
-private:
-    std::optional<BasicDerivation> tryResolveUncached(Store & store);
 };
 
 

From ae4260f0a79c5cbb7c88ddbef1f512e0771f7414 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 15 Feb 2021 10:20:54 +0000
Subject: [PATCH 743/882] Generate installer script for each PR/push

This works by using Cachix feature of serving a file from
a store path.
---
 .github/workflows/test.yml                   | 44 +++++++++++-
 flake.nix                                    | 70 ++++++++++----------
 scripts/prepare-installer-for-github-actions | 10 +++
 3 files changed, 89 insertions(+), 35 deletions(-)
 create mode 100755 scripts/prepare-installer-for-github-actions

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 021642f4c..bde6106e0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,10 +8,52 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    env:
+      CACHIX_NAME: nix-ci
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v12
+    - uses: cachix/cachix-action@v8
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+  installer:
+    if: github.event_name == 'push'
+    needs: tests
+    runs-on: ubuntu-latest
+    env:
+      CACHIX_NAME: nix-ci
+    outputs:
+      installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
+    steps:
+    - uses: actions/checkout@v2.3.4
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v12
+    - uses: cachix/cachix-action@v8
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+    - id: prepare-installer
+      run: scripts/prepare-installer-for-github-actions
+  installer_test:
+    if: github.event_name == 'push'
+    needs: installer
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    env:
+      CACHIX_NAME: nix-ci
+    steps:
+    - uses: actions/checkout@v2.3.4
+    - uses: cachix/install-nix-action@master
+      with:
+        install_url: '${{needs.installer.outputs.installerURL}}'
+        install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve'
+    - run: nix-instantiate -E 'builtins.currentTime' --eval
+    
\ No newline at end of file
diff --git a/flake.nix b/flake.nix
index 8c60934e6..fc334ac5b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -109,6 +109,40 @@
           ];
       };
 
+    installScriptFor = systems:
+      with nixpkgsFor.x86_64-linux;
+        runCommand "installer-script"
+          { buildInputs = [ nix ];
+          }
+          ''
+            mkdir -p $out/nix-support
+
+            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
+            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+            tarballPath() {
+              # Remove the store prefix
+              local path=''${1#${builtins.storeDir}/}
+              # Get the path relative to the derivation root
+              local rest=''${path#*/}
+              # Get the derivation hash
+              local drvHash=''${path%%-*}
+              echo "$drvHash/$rest"
+            }
+
+            substitute ${./scripts/install.in} $out/install \
+              ${pkgs.lib.concatMapStrings
+                (system:
+                  '' \
+                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                  --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                  ''
+                )
+                systems
+              } --replace '@nixVersion@' ${version}
+
+            echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+          '';
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
@@ -313,40 +347,8 @@
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript =
-          with nixpkgsFor.x86_64-linux;
-          runCommand "installer-script"
-            { buildInputs = [ nix ];
-            }
-            ''
-              mkdir -p $out/nix-support
-
-              # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
-              # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
-              tarballPath() {
-                # Remove the store prefix
-                local path=''${1#${builtins.storeDir}/}
-                # Get the path relative to the derivation root
-                local rest=''${path#*/}
-                # Get the derivation hash
-                local drvHash=''${path%%-*}
-                echo "$drvHash/$rest"
-              }
-
-              substitute ${./scripts/install.in} $out/install \
-                ${pkgs.lib.concatMapStrings
-                  (system:
-                    '' \
-                    --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                    --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                    ''
-                  )
-                  [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-                } \
-                --replace '@nixVersion@' ${version}
-
-              echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-            '';
+        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
 
         # Line coverage analysis.
         coverage =
diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions
new file mode 100755
index 000000000..92d930384
--- /dev/null
+++ b/scripts/prepare-installer-for-github-actions
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
+installerHash=$(echo $script | cut -b12-43 -)
+
+installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
+
+echo "::set-output name=installerURL::$installerURL"

From 22aec8cef43e77bba356d099868fe0a6e7545b43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Sun, 21 Feb 2021 14:55:45 +0000
Subject: [PATCH 744/882] fix installer script

---
 scripts/install.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install.in b/scripts/install.in
index 0eaf25bb3..7d25f7bd7 100755
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -60,7 +60,7 @@ case "$(uname -s).$(uname -m)" in
 esac
 
 # Use this command-line option to fetch the tarballs using nar-serve or Cachix
-if "${1:---tarball-url-prefix}"; then
+if [ "${1:-}" = "--tarball-url-prefix" ]; then
     if [ -z "${2:-}" ]; then
         oops "missing argument for --tarball-url-prefix"
     fi

From 2de232d2b301b2f0854b9fa715ab085612c85e00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Tue, 16 Feb 2021 14:32:12 +0100
Subject: [PATCH 745/882] Add x86_64 compute levels as additional system types

When performing distributed builds of machine learning packages, it
would be nice if builders without the required SIMD instructions can
be excluded as build nodes.

Since x86_64 has accumulated a large number of different instruction
set extensions, listing all possible extensions would be unwieldy.
AMD, Intel, Red Hat, and SUSE have recently defined four different
microarchitecture levels that are now part of the x86-64 psABI
supplement and will be used in glibc 2.33:

https://gitlab.com/x86-psABIs/x86-64-ABI
https://lwn.net/Articles/844831/

This change uses libcpuid to detect CPU features and then uses them to
add the supported x86_64 levels to the additional system types. For
example on a Ryzen 3700X:

$ ~/aps/bin/nix -vv --version | grep "Additional system"
Additional system types: i686-linux, x86_64-v1-linux, x86_64-v2-linux, x86_64-v3-linux
---
 Makefile.config.in            |  1 +
 configure.ac                  |  8 ++++
 flake.nix                     |  3 +-
 src/libstore/globals.cc       | 24 +++++++----
 src/libutil/compute-levels.cc | 80 +++++++++++++++++++++++++++++++++++
 src/libutil/compute-levels.hh |  7 +++
 src/libutil/local.mk          |  4 ++
 tests/compute-levels.sh       |  7 +++
 tests/local.mk                |  3 +-
 9 files changed, 126 insertions(+), 11 deletions(-)
 create mode 100644 src/libutil/compute-levels.cc
 create mode 100644 src/libutil/compute-levels.hh
 create mode 100644 tests/compute-levels.sh

diff --git a/Makefile.config.in b/Makefile.config.in
index d1e59e4e7..9d0500e48 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -9,6 +9,7 @@ CXXFLAGS = @CXXFLAGS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
 GTEST_LIBS = @GTEST_LIBS@
+HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
 LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
diff --git a/configure.ac b/configure.ac
index 2047ed8d2..a24287ff6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -218,6 +218,14 @@ LDFLAGS="-lz $LDFLAGS"
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
+# Look for libcpuid.
+if test "$machine_name" = "x86_64"; then
+  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
+  have_libcpuid=1
+  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+fi
+AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
+
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
diff --git a/flake.nix b/flake.nix
index 8c60934e6..3ad7cca97 100644
--- a/flake.nix
+++ b/flake.nix
@@ -91,7 +91,8 @@
             gmock
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+          ++ lib.optional stdenv.isx86_64 libcpuid;
 
         awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
           (aws-sdk-cpp.override {
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 0531aad9f..df07aee9b 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -3,6 +3,7 @@
 #include "archive.hh"
 #include "args.hh"
 #include "abstract-setting-to-json.hh"
+#include "compute-levels.hh"
 
 #include <algorithm>
 #include <map>
@@ -133,24 +134,29 @@ StringSet Settings::getDefaultSystemFeatures()
 
 StringSet Settings::getDefaultExtraPlatforms()
 {
+    StringSet extraPlatforms;
+
     if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1())
-        return StringSet{"i686-linux"};
-#if __APPLE__
+        extraPlatforms.insert("i686-linux");
+
+#if __linux__
+    StringSet levels = computeLevels();
+    for (auto iter = levels.begin(); iter != levels.end(); ++iter)
+        extraPlatforms.insert(*iter + "-linux");
+#elif __APPLE__
     // Rosetta 2 emulation layer can run x86_64 binaries on aarch64
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
         if (std::string{SYSTEM} == "x86_64-darwin")
-            return StringSet{"aarch64-darwin"};
+            extraPlatforms.insert("aarch64-darwin");
         else if (std::string{SYSTEM} == "aarch64-darwin")
-            return StringSet{"x86_64-darwin"};
-        else
-            return StringSet{};
+            extraPlatforms.insert("x86_64-darwin");
     }
 #endif
-    else
-        return StringSet{};
+
+    return extraPlatforms;
 }
 
 bool Settings::isExperimentalFeatureEnabled(const std::string & name)
diff --git a/src/libutil/compute-levels.cc b/src/libutil/compute-levels.cc
new file mode 100644
index 000000000..19eaedfa8
--- /dev/null
+++ b/src/libutil/compute-levels.cc
@@ -0,0 +1,80 @@
+#include "types.hh"
+
+#if HAVE_LIBCPUID
+#include <libcpuid/libcpuid.h>
+#endif
+
+namespace nix {
+
+#if HAVE_LIBCPUID
+
+StringSet computeLevels() {
+    StringSet levels;
+
+    if (!cpuid_present())
+        return levels;
+
+    cpu_raw_data_t raw;
+    cpu_id_t data;
+
+    if (cpuid_get_raw_data(&raw) < 0)
+        return levels;
+
+    if (cpu_identify(&raw, &data) < 0)
+        return levels;
+
+    if (!(data.flags[CPU_FEATURE_CMOV] &&
+            data.flags[CPU_FEATURE_CX8] &&
+            data.flags[CPU_FEATURE_FPU] &&
+            data.flags[CPU_FEATURE_FXSR] &&
+            data.flags[CPU_FEATURE_MMX] &&
+            data.flags[CPU_FEATURE_SSE] &&
+            data.flags[CPU_FEATURE_SSE2]))
+        return levels;
+
+    levels.insert("x86_64-v1");
+
+    if (!(data.flags[CPU_FEATURE_CX16] &&
+            data.flags[CPU_FEATURE_LAHF_LM] &&
+            data.flags[CPU_FEATURE_POPCNT] &&
+            // SSE3
+            data.flags[CPU_FEATURE_PNI] &&
+            data.flags[CPU_FEATURE_SSSE3] &&
+            data.flags[CPU_FEATURE_SSE4_1] &&
+            data.flags[CPU_FEATURE_SSE4_2]))
+        return levels;
+
+    levels.insert("x86_64-v2");
+
+    if (!(data.flags[CPU_FEATURE_AVX] &&
+            data.flags[CPU_FEATURE_AVX2] &&
+            data.flags[CPU_FEATURE_F16C] &&
+            data.flags[CPU_FEATURE_FMA3] &&
+            // LZCNT
+            data.flags[CPU_FEATURE_ABM] &&
+            data.flags[CPU_FEATURE_MOVBE]))
+        return levels;
+
+    levels.insert("x86_64-v3");
+
+    if (!(data.flags[CPU_FEATURE_AVX512F] &&
+            data.flags[CPU_FEATURE_AVX512BW] &&
+            data.flags[CPU_FEATURE_AVX512CD] &&
+            data.flags[CPU_FEATURE_AVX512DQ] &&
+            data.flags[CPU_FEATURE_AVX512VL]))
+        return levels;
+
+    levels.insert("x86_64-v4");
+
+    return levels;
+}
+
+#else
+
+StringSet computeLevels() {
+    return StringSet{};
+}
+
+#endif // HAVE_LIBCPUID
+
+}
diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh
new file mode 100644
index 000000000..8ded295f9
--- /dev/null
+++ b/src/libutil/compute-levels.hh
@@ -0,0 +1,7 @@
+#include "types.hh"
+
+namespace nix {
+
+StringSet computeLevels();
+
+}
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index ae7eb67ad..5341c58e6 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -7,3 +7,7 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+
+ifeq ($(HAVE_LIBCPUID), 1)
+	libutil_LDFLAGS += -lcpuid
+endif
diff --git a/tests/compute-levels.sh b/tests/compute-levels.sh
new file mode 100644
index 000000000..e4322dfa1
--- /dev/null
+++ b/tests/compute-levels.sh
@@ -0,0 +1,7 @@
+source common.sh
+
+if [[ $(uname -ms) = "Linux x86_64" ]]; then
+    # x86_64 CPUs must always support the baseline
+    # microarchitecture level.
+    nix -vv --version | grep -q "x86_64-v1-linux"
+fi
diff --git a/tests/local.mk b/tests/local.mk
index aa8b4f9bf..06be8cec1 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -38,7 +38,8 @@ nix_tests = \
   describe-stores.sh \
   flakes.sh \
   content-addressed.sh \
-  build.sh
+  build.sh \
+  compute-levels.sh
   # parallel.sh
   # build-remote-content-addressed-fixed.sh \
 

From 574eb2be81cc599162722659dcb95f19173c98d1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 22 Feb 2021 15:24:14 +0100
Subject: [PATCH 746/882] Tweak error message

---
 src/libexpr/eval.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7271776eb..e2f2308aa 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1381,10 +1381,10 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             } else if (!i.def) {
                 throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
-nix attempted to evaluate a function as a top level expression; in this case it must have its
-arguments supplied either by default values, or passed explicitly with --arg or --argstr.
-
-https://nixos.org/manual/nix/stable/#ss-functions)", i.name);
+Nix attempted to evaluate a function as a top level expression; in
+this case it must have its arguments supplied either by default
+values, or passed explicitly with '--arg' or '--argstr'. See
+https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
 
             }
         }

From e2f3b2eb42a0ceca36ce00973bd2d49b1a3e6a2c Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 22 Feb 2021 16:13:09 +0100
Subject: [PATCH 747/882] Make missing auto-call arguments throw an eval error

The PR #4240 changed messag of the error that was thrown when an auto-called
function was missing an argument.
However this change also changed the type of the error, from `EvalError`
to a new `MissingArgumentError`. This broke hydra which was relying on
an `EvalError` being thrown.

Make `MissingArgumentError` a subclass of `EvalError` to un-break hydra.
---
 src/libexpr/nixexpr.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index cbe9a45bf..8df8055b3 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -17,7 +17,7 @@ MakeError(ThrownError, AssertionError);
 MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
-MakeError(MissingArgumentError, Error);
+MakeError(MissingArgumentError, EvalError);
 MakeError(RestrictedPathError, Error);
 
 

From 35205e2e922952fc0654260a07fc3191c5afc2cc Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 22 Feb 2021 17:10:55 -0500
Subject: [PATCH 748/882] Warn about instability of plugin API

---
 src/libstore/globals.hh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 1d968ef3e..1254698ca 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -831,6 +831,9 @@ public:
           command, and RegisterSetting to add new nix config settings. See the
           constructors for those types for more details.
 
+          Warning! These APIs are inherently unstable and may change from
+          release to release.
+
           Since these files are loaded into the same address space as Nix
           itself, they must be DSOs compatible with the instance of Nix
           running at the time (i.e. compiled against the same headers, not

From 6fbf3fe636bc1d9a9aba4bacb2a70191c1d6b1a7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 10:48:41 +0100
Subject: [PATCH 749/882] Make the build-hook work with ca derivations

- Pass it the name of the outputs rather than their output paths (as
  these don't exist for ca derivations)
- Get the built output paths from the remote builder
- Register the new received realisations
---
 src/build-remote/build-remote.cc      | 36 +++++++++++++++++++++------
 src/libstore/build/derivation-goal.cc |  9 ++++---
 src/libstore/realisation.hh           |  2 ++
 src/libstore/remote-store.cc          | 16 ++++++++++++
 src/libstore/store-api.hh             |  2 ++
 src/libstore/worker-protocol.hh       |  2 ++
 6 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 5b8ab3387..c2319a3d1 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -248,7 +248,7 @@ connected:
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
-        auto outputs = readStrings<PathSet>(source);
+        auto wantedOutputs = readStrings<StringSet>(source);
 
         AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
 
@@ -273,6 +273,7 @@ connected:
         uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
+        auto outputHashes = staticOutputHashes(*store, drv);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
@@ -280,16 +281,35 @@ connected:
         if (!result.success())
             throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
 
-        StorePathSet missing;
-        for (auto & path : outputs)
-            if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
+        std::set<Realisation> missingRealisations;
+        StorePathSet missingPaths;
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            for (auto & outputName : wantedOutputs) {
+                auto thisOutputHash = outputHashes.at(outputName);
+                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+                if (!store->queryRealisation(thisOutputId)) {
+                    notice("Missing output %s", outputName);
+                    assert(result.builtOutputs.count(thisOutputId));
+                    auto newRealisation = result.builtOutputs.at(thisOutputId);
+                    missingRealisations.insert(newRealisation);
+                    missingPaths.insert(newRealisation.outPath);
+                }
+            }
+        } else {
+            auto outputPaths = drv.outputsAndOptPaths(*store);
+            for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
+                assert(hopefullyOutputPath.second);
+                if (!store->isValidPath(*hopefullyOutputPath.second))
+                    missingPaths.insert(*hopefullyOutputPath.second);
+            }
+        }
 
-        if (!missing.empty()) {
+        if (!missingPaths.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
-                for (auto & i : missing)
-                    localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
+                for (auto & path : missingPaths)
+                    localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
+            copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
         return 0;
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index d8a89a2d0..b074410b0 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1159,13 +1159,14 @@ HookReply DerivationGoal::tryBuildHook()
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
     {
-        StorePathSet missingPaths;
-        for (auto & [_, status] : initialOutputs) {
+        StringSet missingOutputs;
+        for (auto & [outputName, status] : initialOutputs) {
             if (!status.known) continue;
             if (buildMode != bmCheck && status.known->isValid()) continue;
-            missingPaths.insert(status.known->path);
+            missingOutputs.insert(outputName);
+            /* missingPaths.insert(status.known->path); */
         }
-        worker_proto::write(worker.store, hook->sink, missingPaths);
+        worker_proto::write(worker.store, hook->sink, missingOutputs);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 7c91d802a..fc92d3c17 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -33,6 +33,8 @@ struct Realisation {
     GENERATE_CMP(Realisation, me->id, me->outPath);
 };
 
+typedef std::map<DrvOutput, Realisation> DrvOutputs;
+
 struct OpaquePath {
     StorePath path;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index be07f02dc..52d633372 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -12,6 +12,7 @@
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -49,6 +50,21 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
     out << renderContentAddress(ca);
 }
 
+Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
+{
+    std::string rawInput = readString(from);
+    return Realisation::fromJSON(
+        nlohmann::json::parse(rawInput),
+        "remote-protocol"
+    );
+}
+void write(const Store & store, Sink & out, const Realisation & realisation)
+{ out << realisation.toJSON().dump(); }
+
+DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
+{ return DrvOutput::parse(readString(from)); }
+void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+{ out << drvOutput.to_string(); }
 
 std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
 {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6dcd43ed1..ea6389ba4 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -162,6 +162,8 @@ struct BuildResult
        non-determinism.) */
     bool isNonDeterministic = false;
 
+    DrvOutputs builtOutputs;
+
     /* The start/stop times of the build (or one of the rounds, if it
        was repeated). */
     time_t startTime = 0, stopTime = 0;
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index f2cdc7ca3..5e094c378 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -86,6 +86,8 @@ namespace worker_proto {
 MAKE_WORKER_PROTO(, std::string);
 MAKE_WORKER_PROTO(, StorePath);
 MAKE_WORKER_PROTO(, ContentAddress);
+MAKE_WORKER_PROTO(, Realisation);
+MAKE_WORKER_PROTO(, DrvOutput);
 
 MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
 

From 5687564a27bee692f68a78b897a2d68715f6a3ce Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 10:50:44 +0100
Subject: [PATCH 750/882] LocalStore: Send back the new realisations

To allow it to build ca derivations remotely
---
 src/libstore/build/entry-points.cc | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 9f97d40ba..99b3fa070 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -58,6 +58,26 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
         result.status = BuildResult::MiscFailure;
         result.errorMsg = e.msg();
     }
+    // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
+    // extended to return the full realisation for each output
+    auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
+    auto outputHashes = staticOutputHashes(*this, drv);
+    for (auto & [outputName, staticOutput] : staticDrvOutputs) {
+        auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
+        if (staticOutput.second)
+            result.builtOutputs.insert_or_assign(
+                    outputId,
+                    Realisation{ outputId, *staticOutput.second}
+                    );
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            auto realisation = this->queryRealisation(outputId);
+            if (realisation)
+                result.builtOutputs.insert_or_assign(
+                        outputId,
+                        *realisation
+                        );
+        }
+    }
 
     return result;
 }

From a2b69660a9b326b95d48bd222993c5225bbd5b5f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 10:50:44 +0100
Subject: [PATCH 751/882] LegacySSHStore: Send back the new realisations

To allow it to build ca derivations remotely
---
 src/libstore/legacy-ssh-store.cc | 4 +++-
 src/libstore/serve-protocol.hh   | 2 +-
 src/nix-store/nix-store.cc       | 4 ++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 253c0033e..daf78042f 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -258,7 +258,9 @@ public:
 
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
-
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
+            status.builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+        }
         return status;
     }
 
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index 9fae6d534..0a17387cb 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -5,7 +5,7 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
-#define SERVE_PROTOCOL_VERSION 0x205
+#define SERVE_PROTOCOL_VERSION 0x206
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 37191b9e6..559fd5355 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -905,6 +905,10 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
                     out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
+                if (GET_PROTOCOL_MINOR(clientVersion >= 5)) {
+                    worker_proto::write(*store, out, status.builtOutputs);
+                }
+
 
                 break;
             }

From 27b5747ca7b5599768083dde5fa4d36bfbb0f66f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 25 Jan 2021 11:08:38 +0100
Subject: [PATCH 752/882] RemoteStore: Send back the new realisations

To allow it to build ca derivations remotely
---
 src/libstore/daemon.cc          | 3 +++
 src/libstore/remote-store.cc    | 4 ++++
 src/libstore/worker-protocol.hh | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index ba5788b64..ba7959263 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -575,6 +575,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
         to << res.status << res.errorMsg;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) {
+            worker_proto::write(*store, to, res.builtOutputs);
+        }
         break;
     }
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 52d633372..0d884389a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -680,6 +680,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     unsigned int status;
     conn->from >> status >> res.errorMsg;
     res.status = (BuildResult::Status) status;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) {
+        auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+        res.builtOutputs = builtOutputs;
+    }
     return res;
 }
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 5e094c378..95f08bc9a 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x11b
+#define PROTOCOL_VERSION 0x11c
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

From 8c385d16eeeb26a912d213c5689d9f9a78020bc7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 09:35:10 +0100
Subject: [PATCH 753/882] Also send ca outputs to the build hook

Otherwise they don't get registered, triggering an assertion failure
at some point later
---
 src/libstore/build/derivation-goal.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index b074410b0..096f24029 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1161,8 +1161,8 @@ HookReply DerivationGoal::tryBuildHook()
     {
         StringSet missingOutputs;
         for (auto & [outputName, status] : initialOutputs) {
-            if (!status.known) continue;
-            if (buildMode != bmCheck && status.known->isValid()) continue;
+            // XXX: Does this include known CA outputs?
+            if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
             /* missingPaths.insert(status.known->path); */
         }

From 69666b951ee06733ed420cb4cd408a19e42c6e43 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 09:36:24 +0100
Subject: [PATCH 754/882] build-remote: Always register the missing outputs

It's possible that all the paths are already there, but just not
associated to the current drv output
---
 src/build-remote/build-remote.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index c2319a3d1..228aba35a 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -311,6 +311,13 @@ connected:
                     localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
         }
+        // XXX: Should e done as part of `copyPaths`
+        for (auto & realisation : missingRealisations) {
+            // Should hold, because if the feature isn't enabled the set
+            // of missing realisations should be empty
+            settings.requireExperimentalFeature("ca-derivations");
+            store->registerDrvOutput(realisation);
+        }
 
         return 0;
     }

From 527da736905730e70725bf4b3556d61267d220ba Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 10:02:03 +0100
Subject: [PATCH 755/882] Properly bypass the registering step when all outputs
 are present

There was already some logic for that, but it didn't handle the case of
content-addressed outputs, so extend it a bit for that
---
 src/libstore/build/derivation-goal.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 096f24029..6052b625d 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -3001,11 +3001,11 @@ void DerivationGoal::registerOutputs()
      */
     if (hook) {
         bool allValid = true;
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (!i.second.second || !worker.store.isValidPath(*i.second.second))
+        for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
+            if (!outputPath || !worker.store.isValidPath(*outputPath))
                 allValid = false;
             else
-                finalOutputs.insert_or_assign(i.first, *i.second.second);
+                finalOutputs.insert_or_assign(outputName, *outputPath);
         }
         if (allValid) return;
     }

From c32168c9bc161e0c9cea027853895971699510cb Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 26 Jan 2021 10:28:00 +0100
Subject: [PATCH 756/882] Test the remote building of ca derivations

---
 tests/build-hook-ca.nix                          | 16 ++++++++++++----
 tests/build-remote-content-addressed-fixed.sh    |  5 -----
 tests/build-remote-content-addressed-floating.sh |  7 +++++++
 tests/local.mk                                   |  2 +-
 4 files changed, 20 insertions(+), 10 deletions(-)
 delete mode 100644 tests/build-remote-content-addressed-fixed.sh
 create mode 100644 tests/build-remote-content-addressed-floating.sh

diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca.nix
index 98db473fc..67295985f 100644
--- a/tests/build-hook-ca.nix
+++ b/tests/build-hook-ca.nix
@@ -11,6 +11,7 @@ let
       args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       outputHashMode = "recursive";
       outputHashAlgo = "sha256";
+      __contentAddressed = true;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 
@@ -19,7 +20,6 @@ let
     name = "build-remote-input-1";
     buildCommand = "echo FOO > $out";
     requiredSystemFeatures = ["foo"];
-    outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=";
   };
 
   input2 = mkDerivation {
@@ -27,7 +27,16 @@ let
     name = "build-remote-input-2";
     buildCommand = "echo BAR > $out";
     requiredSystemFeatures = ["bar"];
-    outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q=";
+  };
+
+  input3 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-3";
+    buildCommand = ''
+      read x < ${input2}
+      echo $x BAZ > $out
+    '';
+    requiredSystemFeatures = ["baz"];
   };
 
 in
@@ -38,8 +47,7 @@ in
     buildCommand =
       ''
         read x < ${input1}
-        read y < ${input2}
+        read y < ${input3}
         echo "$x $y" > $out
       '';
-    outputHash = "sha256-3YGhlOfbGUm9hiPn2teXXTT8M1NEpDFvfXkxMaJRld0=";
   }
diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh
deleted file mode 100644
index 1408a19d5..000000000
--- a/tests/build-remote-content-addressed-fixed.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-source common.sh
-
-file=build-hook-ca.nix
-
-source build-remote.sh
diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh
new file mode 100644
index 000000000..cbb75729b
--- /dev/null
+++ b/tests/build-remote-content-addressed-floating.sh
@@ -0,0 +1,7 @@
+source common.sh
+
+file=build-hook-ca.nix
+
+sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
+
+source build-remote.sh
diff --git a/tests/local.mk b/tests/local.mk
index aa8b4f9bf..9bde2322f 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -17,6 +17,7 @@ nix_tests = \
   linux-sandbox.sh \
   build-dry.sh \
   build-remote-input-addressed.sh \
+  build-remote-content-addressed-floating.sh \
   ssh-relay.sh \
   nar-access.sh \
   structured-attrs.sh \
@@ -40,7 +41,6 @@ nix_tests = \
   content-addressed.sh \
   build.sh
   # parallel.sh
-  # build-remote-content-addressed-fixed.sh \
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 

From ba1a256d0875592b28d902f3e40663b2adedfe9c Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 23 Feb 2021 14:12:11 +0100
Subject: [PATCH 757/882] Make `DerivationGoal::drv` a full Derivation

This field used to be a `BasicDerivation`, but this `BasicDerivation`
was downcasted to a `Derivation` when needed (implicitely or not), so we
might as well make it a full `Derivation` and upcast it when needed.

This also allows getting rid of a weird duplication in the way we
compute the static output hashes for the derivation. We had to
do it differently and in a different place depending on whether the
derivation was a full derivation or just a basic drv, but we can now do
it unconditionally on the full derivation.

Fix #4559
---
 src/libstore/build/derivation-goal.cc | 37 ++++++++++-----------------
 src/libstore/build/derivation-goal.hh |  2 +-
 2 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index d8a89a2d0..804a79e4c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -123,17 +123,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
-
-    auto outputHashes = staticOutputHashes(worker.store, drv);
-    for (auto &[outputName, outputHash] : outputHashes)
-      initialOutputs.insert({
-            outputName,
-            InitialOutput{
-                .wanted = true, // Will be refined later
-                .outputHash = outputHash
-            }
-          });
+    this->drv = std::make_unique<Derivation>(drv);
 
     state = &DerivationGoal::haveDerivation;
     name = fmt(
@@ -271,18 +261,8 @@ void DerivationGoal::loadDerivation()
 
     auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath));
 
-    auto outputHashes = staticOutputHashes(worker.store, *fullDrv);
-    for (auto &[outputName, outputHash] : outputHashes)
-      initialOutputs.insert({
-            outputName,
-            InitialOutput{
-                .wanted = true, // Will be refined later
-                .outputHash = outputHash
-            }
-          });
-
     /* Get the derivation. */
-    drv = std::unique_ptr<BasicDerivation>(fullDrv);
+    drv = std::unique_ptr<Derivation>(fullDrv);
 
     haveDerivation();
 }
@@ -301,6 +281,16 @@ void DerivationGoal::haveDerivation()
         if (i.second.second)
             worker.store.addTempRoot(*i.second.second);
 
+    auto outputHashes = staticOutputHashes(worker.store, *drv);
+    for (auto &[outputName, outputHash] : outputHashes)
+      initialOutputs.insert({
+            outputName,
+            InitialOutput{
+                .wanted = true, // Will be refined later
+                .outputHash = outputHash
+            }
+          });
+
     /* Check what outputs paths are not already valid. */
     checkPathValidity();
     bool allValid = true;
@@ -3517,10 +3507,9 @@ void DerivationGoal::registerOutputs()
        but it's fine to do in all cases. */
 
     if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-        auto outputHashes = staticOutputHashes(worker.store, *drv);
         for (auto& [outputName, newInfo] : infos)
             worker.store.registerDrvOutput(Realisation{
-                .id = DrvOutput{outputHashes.at(outputName), outputName},
+                .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName},
                 .outPath = newInfo.path});
     }
 }
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 761100d3a..6dc164922 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -64,7 +64,7 @@ struct DerivationGoal : public Goal
     bool retrySubstitution;
 
     /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
+    std::unique_ptr<Derivation> drv;
 
     std::unique_ptr<ParsedDerivation> parsedDrv;
 

From ec3497c1d63f4c0547d0402d92015f846f56aac7 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 28 Jan 2021 07:37:04 -0500
Subject: [PATCH 758/882] Bail if plugin-files is set after plugins have been
 loaded.

We know the flag will be ignored but the user wants it to take effect.
---
 src/libstore/globals.cc | 11 +++++++++++
 src/libstore/globals.hh | 19 ++++++++++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index df07aee9b..03294b7fe 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -243,6 +243,14 @@ void MaxBuildJobsSetting::set(const std::string & str, bool append)
 }
 
 
+void PluginFilesSetting::set(const std::string & str, bool append)
+{
+    if (pluginsLoaded)
+        throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand");
+    BaseSetting<Paths>::set(str, append);
+}
+
+
 void initPlugins()
 {
     for (const auto & pluginFile : settings.pluginFiles.get()) {
@@ -270,6 +278,9 @@ void initPlugins()
        unknown settings. */
     globalConfig.reapplyUnknownSettings();
     globalConfig.warnUnknownSettings();
+
+    /* Tell the user if they try to set plugin-files after we've already loaded */
+    settings.pluginFiles.pluginsLoaded = true;
 }
 
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 1254698ca..df61d6417 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -28,6 +28,23 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
     void set(const std::string & str, bool append = false) override;
 };
 
+struct PluginFilesSetting : public BaseSetting<Paths>
+{
+    bool pluginsLoaded = false;
+
+    PluginFilesSetting(Config * options,
+        const Paths & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : BaseSetting<Paths>(def, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    void set(const std::string & str, bool append = false) override;
+};
+
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -819,7 +836,7 @@ public:
     Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
         "Number of seconds between checking free disk space."};
 
-    Setting<Paths> pluginFiles{
+    PluginFilesSetting pluginFiles{
         this, {}, "plugin-files",
         R"(
           A list of plugin files to be loaded by Nix. Each of these files will

From 98d1b64400cc7b75216fc885859883c707c18bef Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 28 Jan 2021 09:37:43 -0500
Subject: [PATCH 759/882] Initialize plugins after handling initial command
 line flags

This is technically a breaking change, since attempting to set plugin
files after the first non-flag argument will now throw an error. This
is acceptable given the relative lack of stability in a plugin
interface and the need to tie the knot somewhere once plugins can
actually define new subcommands.
---
 doc/manual/src/release-notes/rl-2.4.md         | 7 +++++++
 src/build-remote/build-remote.cc               | 3 +++
 src/libmain/common-args.cc                     | 7 +++++++
 src/libmain/common-args.hh                     | 6 +++++-
 src/libstore/globals.cc                        | 1 +
 src/libutil/args.cc                            | 8 ++++++++
 src/libutil/args.hh                            | 4 ++++
 src/nix-build/nix-build.cc                     | 2 --
 src/nix-channel/nix-channel.cc                 | 2 --
 src/nix-collect-garbage/nix-collect-garbage.cc | 2 --
 src/nix-copy-closure/nix-copy-closure.cc       | 2 --
 src/nix-env/nix-env.cc                         | 2 --
 src/nix-instantiate/nix-instantiate.cc         | 2 --
 src/nix-store/nix-store.cc                     | 2 --
 src/nix/daemon.cc                              | 2 --
 src/nix/main.cc                                | 2 --
 src/nix/prefetch.cc                            | 2 --
 tests/plugins.sh                               | 2 +-
 18 files changed, 36 insertions(+), 22 deletions(-)
 create mode 100644 doc/manual/src/release-notes/rl-2.4.md

diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md
new file mode 100644
index 000000000..26ba70904
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.4.md
@@ -0,0 +1,7 @@
+# Release 2.4 (202X-XX-XX)
+
+  - It is now an error to modify the `plugin-files` setting via a
+    command-line flag that appears after the first non-flag argument
+    to any command, including a subcommand to `nix`. For example,
+    `nix-instantiate default.nix --plugin-files ""` must now become
+    `nix-instantiate --plugin-files "" default.nix`.
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 5b8ab3387..f784b5160 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -53,6 +53,9 @@ static int main_build_remote(int argc, char * * argv)
         unsetenv("DISPLAY");
         unsetenv("SSH_ASKPASS");
 
+        /* If we ever use the common args framework, make sure to
+           remove initPlugins below and initialize settings first.
+        */
         if (argc != 2)
             throw UsageError("called without required arguments");
 
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index ff96ee7d5..c43e9ebd2 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -79,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     hiddenCategories.insert(cat);
 }
 
+void MixCommonArgs::initialFlagsProcessed()
+{
+    initPlugins();
+    pluginsInited();
+}
+
+
 }
diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh
index 8e53a7361..31bdf527a 100644
--- a/src/libmain/common-args.hh
+++ b/src/libmain/common-args.hh
@@ -7,10 +7,14 @@ namespace nix {
 //static constexpr auto commonArgsCategory = "Miscellaneous common options";
 static constexpr auto loggingCategory = "Logging-related options";
 
-struct MixCommonArgs : virtual Args
+class MixCommonArgs : public virtual Args
 {
+    void initialFlagsProcessed() override;
+public:
     string programName;
     MixCommonArgs(const string & programName);
+protected:
+    virtual void pluginsInited() {}
 };
 
 struct MixDryRun : virtual Args
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 03294b7fe..2780e0bf5 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -253,6 +253,7 @@ void PluginFilesSetting::set(const std::string & str, bool append)
 
 void initPlugins()
 {
+    assert(!settings.pluginFiles.pluginsLoaded);
     for (const auto & pluginFile : settings.pluginFiles.get()) {
         Paths pluginFiles;
         try {
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 9377fe4c0..eb11fd64b 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -60,6 +60,7 @@ void Args::parseCmdline(const Strings & _cmdline)
         verbosity = lvlError;
     }
 
+    bool argsSeen = false;
     for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
 
         auto arg = *pos;
@@ -88,6 +89,10 @@ void Args::parseCmdline(const Strings & _cmdline)
                 throw UsageError("unrecognised flag '%1%'", arg);
         }
         else {
+            if (!argsSeen) {
+                argsSeen = true;
+                initialFlagsProcessed();
+            }
             pos = rewriteArgs(cmdline, pos);
             pendingArgs.push_back(*pos++);
             if (processArgs(pendingArgs, false))
@@ -96,6 +101,9 @@ void Args::parseCmdline(const Strings & _cmdline)
     }
 
     processArgs(pendingArgs, true);
+
+    if (!argsSeen)
+        initialFlagsProcessed();
 }
 
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 88f068087..4721c21df 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -132,6 +132,10 @@ protected:
 
     std::set<std::string> hiddenCategories;
 
+    /* Called after all command line flags before the first non-flag
+       argument (if any) have been processed. */
+    virtual void initialFlagsProcessed() {}
+
 public:
 
     void addFlag(Flag && flag);
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index d975cd16d..7b4a53919 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -240,8 +240,6 @@ static void main_nix_build(int argc, char * * argv)
 
     myArgs.parseCmdline(args);
 
-    initPlugins();
-
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 57189d557..3272c6125 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -196,8 +196,6 @@ static int main_nix_channel(int argc, char ** argv)
             return true;
         });
 
-        initPlugins();
-
         switch (cmd) {
             case cNone:
                 throw UsageError("no command specified");
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index c1769790a..4f953fab4 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -74,8 +74,6 @@ static int main_nix_collect_garbage(int argc, char * * argv)
             return true;
         });
 
-        initPlugins();
-
         auto profilesDir = settings.nixStateDir + "/profiles";
         if (removeOld) removeOldGenerations(profilesDir);
 
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index ad2e06067..5e8cc515b 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -43,8 +43,6 @@ static int main_nix_copy_closure(int argc, char ** argv)
             return true;
         });
 
-        initPlugins();
-
         if (sshHost.empty())
             throw UsageError("no host name specified");
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 106a78fc4..0f10a4cbb 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1420,8 +1420,6 @@ static int main_nix_env(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
-        initPlugins();
-
         if (!op) throw UsageError("no operation specified");
 
         auto store = openStore();
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index ea2e85eb0..95903d882 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -149,8 +149,6 @@ static int main_nix_instantiate(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
-        initPlugins();
-
         if (evalOnly && !wantsReadWrite)
             settings.readOnlyMode = true;
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 37191b9e6..e17b38c3c 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -1067,8 +1067,6 @@ static int main_nix_store(int argc, char * * argv)
             return true;
         });
 
-        initPlugins();
-
         if (!op) throw UsageError("no operation specified");
 
         if (op != opDump && op != opRestore) /* !!! hack */
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 26006167d..2cf2a04c9 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -326,8 +326,6 @@ static int main_nix_daemon(int argc, char * * argv)
             return true;
         });
 
-        initPlugins();
-
         runDaemon(stdio);
 
         return 0;
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 1b68cf15b..b078366fa 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -283,8 +283,6 @@ void mainWrapped(int argc, char * * argv)
 
     if (completions) return;
 
-    initPlugins();
-
     if (args.showVersion) {
         printVersion(programName);
         return;
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index a831dcd15..b7da3ea5a 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -171,8 +171,6 @@ static int main_nix_prefetch_url(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
-        initPlugins();
-
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
diff --git a/tests/plugins.sh b/tests/plugins.sh
index 50bfaf7e9..e22bf4408 100644
--- a/tests/plugins.sh
+++ b/tests/plugins.sh
@@ -2,6 +2,6 @@ source common.sh
 
 set -o pipefail
 
-res=$(nix eval --expr builtins.anotherNull --option setting-set true --option plugin-files $PWD/plugins/libplugintest*)
+res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull)
 
 [ "$res"x = "nullx" ]

From f6c5b05488c588964f51ce97ad2c297fbca7ce96 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 28 Jan 2021 10:04:47 -0500
Subject: [PATCH 760/882] Respect command registrations in plugins.

---
 doc/manual/src/release-notes/rl-2.4.md | 1 +
 src/libutil/args.cc                    | 4 ++--
 src/nix/main.cc                        | 6 ++++++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md
index 26ba70904..f7ab9f6ad 100644
--- a/doc/manual/src/release-notes/rl-2.4.md
+++ b/doc/manual/src/release-notes/rl-2.4.md
@@ -5,3 +5,4 @@
     to any command, including a subcommand to `nix`. For example,
     `nix-instantiate default.nix --plugin-files ""` must now become
     `nix-instantiate --plugin-files "" default.nix`.
+  - Plugins that add new `nix` subcommands are now actually respected.
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index eb11fd64b..75eb19d28 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -306,8 +306,8 @@ Strings argvToStrings(int argc, char * * argv)
     return args;
 }
 
-MultiCommand::MultiCommand(const Commands & commands)
-    : commands(commands)
+MultiCommand::MultiCommand(const Commands & commands_)
+    : commands(commands_)
 {
     expectArgs({
         .label = "subcommand",
diff --git a/src/nix/main.cc b/src/nix/main.cc
index b078366fa..06e221682 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -159,6 +159,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
           #include "nix.md"
           ;
     }
+
+    // Plugins may add new subcommands.
+    void pluginsInited() override
+    {
+        commands = RegisterCommand::getCommandsFor({});
+    }
 };
 
 static void showHelp(std::vector<std::string> subcommand)

From 1130b2882415b003f5ba2fc0b5466b573fe1b05a Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 24 Feb 2021 20:52:22 -0500
Subject: [PATCH 761/882] distributed builds: load remote builder host key from
 the machines file

This is already used by Hydra, and is very useful when materializing
a remote builder list from service discovery. This allows the service
discovery tool to only sync one file instead of two.
---
 .../src/advanced-topics/distributed-builds.md    | 10 +++++++---
 src/libstore/legacy-ssh-store.cc                 |  2 ++
 src/libstore/machines.cc                         |  6 ++++++
 src/libstore/ssh-store.cc                        |  2 ++
 src/libstore/ssh.cc                              | 16 ++++++++++++++--
 src/libstore/ssh.hh                              |  3 ++-
 6 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
index c6966a50b..580b36736 100644
--- a/doc/manual/src/advanced-topics/distributed-builds.md
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -37,7 +37,7 @@ then you need to ensure that the `PATH` of non-interactive login shells
 contains Nix.
 
 > **Warning**
-> 
+>
 > If you are building via the Nix daemon, it is the Nix daemon user
 > account (that is, `root`) that should have SSH access to the remote
 > machine. If you can’t or don’t want to configure `root` to be able to
@@ -52,7 +52,7 @@ example, the following command allows you to build a derivation for
 ```console
 $ uname
 Linux
-    
+
 $ nix build \
   '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
@@ -103,7 +103,7 @@ default, set it to `-`.
     ```nix
     requiredSystemFeatures = [ "kvm" ];
     ```
-    
+
     will cause the build to be performed on a machine that has the `kvm`
     feature.
 
@@ -112,6 +112,10 @@ default, set it to `-`.
     features appear in the derivation’s `requiredSystemFeatures`
     attribute..
 
+8.  The (base64-encoded) public host key of the remote machine. If omitted, SSH
+    will use its regular known-hosts file. Specifically, the field is calculated
+    via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`.
+
 For example, the machine specification
 
     nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 253c0033e..99b0bb5a8 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -15,6 +15,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
     using StoreConfig::StoreConfig;
     const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
     const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
     const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
     const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@@ -59,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         , master(
             host,
             sshKey,
+            sshPublicHostKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
             compress,
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index 7db2556f4..b42e5e434 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -54,9 +54,15 @@ ref<Store> Machine::openStore() const {
     if (hasPrefix(storeUri, "ssh://")) {
         storeParams["max-connections"] = "1";
         storeParams["log-fd"] = "4";
+    }
+
+    if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
         if (sshKey != "")
             storeParams["ssh-key"] = sshKey;
+        if (sshPublicHostKey != "")
+            storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
     }
+
     {
         auto & fs = storeParams["system-features"];
         auto append = [&](auto feats) {
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 17c258201..f2caf2aeb 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -13,6 +13,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
     using RemoteStoreConfig::RemoteStoreConfig;
 
     const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
     const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
     const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@@ -34,6 +35,7 @@ public:
         , master(
             host,
             sshKey,
+            sshPublicHostKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
             compress)
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 84548a6e4..235eed37a 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -2,24 +2,37 @@
 
 namespace nix {
 
-SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
+SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD)
     : host(host)
     , fakeSSH(host == "localhost")
     , keyFile(keyFile)
+    , sshPublicHostKey(sshPublicHostKey)
     , useMaster(useMaster && !fakeSSH)
     , compress(compress)
     , logFD(logFD)
 {
     if (host == "" || hasPrefix(host, "-"))
         throw Error("invalid SSH host name '%s'", host);
+
+    auto state(state_.lock());
+    state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
 }
 
 void SSHMaster::addCommonSSHOpts(Strings & args)
 {
+    auto state(state_.lock());
+
     for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS").value_or("")))
         args.push_back(i);
     if (!keyFile.empty())
         args.insert(args.end(), {"-i", keyFile});
+    if (!sshPublicHostKey.empty()) {
+        Path fileName = (Path) *state->tmpDir + "/host-key";
+        auto p = host.rfind("@");
+        string thost = p != string::npos ? string(host, p + 1) : host;
+        writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n");
+        args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName});
+    }
     if (compress)
         args.push_back("-C");
 }
@@ -87,7 +100,6 @@ Path SSHMaster::startMaster()
 
     if (state->sshMaster != -1) return state->socketPath;
 
-    state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
 
     state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
 
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 4f0f0bd29..dabbcedda 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -12,6 +12,7 @@ private:
     const std::string host;
     bool fakeSSH;
     const std::string keyFile;
+    const std::string sshPublicHostKey;
     const bool useMaster;
     const bool compress;
     const int logFD;
@@ -29,7 +30,7 @@ private:
 
 public:
 
-    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
+    SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1);
 
     struct Connection
     {

From 2e199673a523fa81de31ffdd2a25976ce0814631 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 14 Dec 2020 19:43:53 +0100
Subject: [PATCH 762/882] Use `RealisedPath`s in `copyPaths`

That way we can copy the realisations too (in addition to the store
paths themselves)
---
 src/libstore/store-api.cc                | 31 ++++++++++++++----------
 src/libstore/store-api.hh                |  9 +++----
 src/nix-copy-closure/nix-copy-closure.cc |  6 ++---
 src/nix/copy.cc                          | 13 +++++-----
 4 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 2658f7617..529c34de5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -783,6 +783,24 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 }
 
 
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
+    RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
+{
+    StorePathSet storePaths;
+    std::set<Realisation> realisations;
+    for (auto path : paths) {
+        storePaths.insert(path.path());
+        if (auto realisation = std::get_if<Realisation>(&path.raw))
+            realisations.insert(*realisation);
+    }
+    auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+    for (auto& realisation : realisations) {
+        dstStore->registerDrvOutput(realisation);
+    }
+
+    return pathsMap;
+}
+
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
@@ -796,7 +814,6 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     for (auto & path : storePaths)
         pathsMap.insert_or_assign(path, path);
 
-    if (missing.empty()) return pathsMap;
 
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
@@ -871,21 +888,9 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
             nrDone++;
             showProgress();
         });
-
     return pathsMap;
 }
 
-
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
-    SubstituteFlag substitute)
-{
-    StorePathSet closure;
-    srcStore->computeFSClosure(storePaths, closure);
-    copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
-}
-
-
 std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
 {
     std::string path;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6dcd43ed1..63b26422a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -752,15 +752,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
    that. Returns a map of what each path was copied to the dstStore
    as. */
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths,
+    const RealisedPath::Set&,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
-
-
-/* Copy the closure of the specified paths from one store to another. */
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
+    const StorePathSet& paths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 5e8cc515b..02ccbe541 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -50,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv)
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
 
-        StorePathSet storePaths2;
+        RealisedPath::Set storePaths2;
         for (auto & path : storePaths)
             storePaths2.insert(from->followLinksToStorePath(path));
 
-        StorePathSet closure;
-        from->computeFSClosure(storePaths2, closure, false, includeOutputs);
+        RealisedPath::Set closure;
+        RealisedPath::closure(*from, storePaths2, closure);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index c56a1def1..f59f7c76b 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -8,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdCopy : StorePathsCommand
+struct CmdCopy : RealisedPathsCommand
 {
     std::string srcUri, dstUri;
 
@@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand
 
     SubstituteFlag substitute = NoSubstitute;
 
-    using StorePathsCommand::run;
+    using RealisedPathsCommand::run;
 
     CmdCopy()
-        : StorePathsCommand(true)
+        : RealisedPathsCommand(true)
     {
         addFlag({
             .longName = "from",
@@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand
         if (srcUri.empty() && dstUri.empty())
             throw UsageError("you must pass '--from' and/or '--to'");
 
-        StorePathsCommand::run(store);
+        RealisedPathsCommand::run(store);
     }
 
-    void run(ref<Store> srcStore, StorePaths storePaths) override
+    void run(ref<Store> srcStore, std::vector<RealisedPath> paths) override
     {
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
-        copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),
+        copyPaths(
+            srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()),
             NoRepair, checkSigs, substitute);
     }
 };

From aead35531a0630b19e41348e103b2d105e2d8dd9 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 15 Dec 2020 09:37:05 +0100
Subject: [PATCH 763/882] Add a test for the copy of CA paths

---
 tests/local.mk                      |  1 +
 tests/nix-copy-content-addressed.sh | 34 +++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100755 tests/nix-copy-content-addressed.sh

diff --git a/tests/local.mk b/tests/local.mk
index 06be8cec1..a504e397e 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -38,6 +38,7 @@ nix_tests = \
   describe-stores.sh \
   flakes.sh \
   content-addressed.sh \
+  nix-copy-content-addressed.sh \
   build.sh \
   compute-levels.sh
   # parallel.sh
diff --git a/tests/nix-copy-content-addressed.sh b/tests/nix-copy-content-addressed.sh
new file mode 100755
index 000000000..2e0dea2d2
--- /dev/null
+++ b/tests/nix-copy-content-addressed.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+# Globally enable the ca derivations experimental flag
+sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf"
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+ensureCorrectlyCopied () {
+    attrPath="$1"
+    nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath"
+}
+
+testOneCopy () {
+    clearStore
+    rm -rf "$REMOTE_STORE_DIR"
+
+    attrPath="$1"
+    nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix
+
+    ensureCorrectlyCopied "$attrPath"
+
+    # Ensure that we can copy back what we put in the store
+    clearStore
+    nix copy --from $REMOTE_STORE \
+        --file ./content-addressed.nix "$attrPath" \
+        --no-check-sigs
+}
+
+for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do
+    testOneCopy "$attrPath"
+done

From f67ff1f5756018387a2d23c8f6772580192d30ad Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 19 Feb 2021 17:58:28 +0100
Subject: [PATCH 764/882] Don't crash when copying realisations to a non-ca
 remote

Rather throw a proper exception, and catch&log it on the client side
---
 src/libstore/globals.cc     |  7 ++++++-
 src/libstore/globals.hh     | 18 ++++++++++++++----
 src/libstore/local-store.cc |  1 +
 src/libstore/store-api.cc   | 14 ++++++++++++--
 4 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 2780e0bf5..8d44003f4 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -165,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
     return std::find(f.begin(), f.end(), name) != f.end();
 }
 
+MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
+    : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
+    , missingFeature(feature)
+    {}
+
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
+        throw MissingExperimentalFeature(name);
 }
 
 bool Settings::isWSL1()
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index df61d6417..25351f55c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -45,6 +45,16 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
+/* MakeError(MissingExperimentalFeature, Error); */
+class MissingExperimentalFeature: public Error
+{
+public:
+    std::string missingFeature;
+
+    MissingExperimentalFeature(std::string feature);
+    virtual const char* sname() const override { return "MissingExperimentalFeature"; }
+};
+
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -632,7 +642,7 @@ public:
           is `root`.
 
           > **Warning**
-          > 
+          >
           > Adding a user to `trusted-users` is essentially equivalent to
           > giving that user root access to the system. For example, the user
           > can set `sandbox-paths` and thereby obtain read access to
@@ -722,13 +732,13 @@ public:
           The program executes with no arguments. The program's environment
           contains the following environment variables:
 
-            - `DRV_PATH`  
+            - `DRV_PATH`
               The derivation for the built paths.
 
               Example:
               `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
 
-            - `OUT_PATHS`  
+            - `OUT_PATHS`
               Output paths of the built derivation, separated by a space
               character.
 
@@ -759,7 +769,7 @@ public:
           documentation](https://ec.haxx.se/usingcurl-netrc.html).
 
           > **Note**
-          > 
+          >
           > This must be an absolute path, and `~` is not resolved. For
           > example, `~/.netrc` won't resolve to your home directory's
           > `.netrc`.
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0962418dd..90fb4a4bd 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
+    settings.requireExperimentalFeature("ca-derivations");
     auto state(_state.lock());
     retrySQLite<void>([&]() {
         state->stmts->RegisterRealisedOutput.use()
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 529c34de5..ac1d8ee2c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -794,8 +794,18 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
             realisations.insert(*realisation);
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
-    for (auto& realisation : realisations) {
-        dstStore->registerDrvOutput(realisation);
+    try {
+        for (auto& realisation : realisations) {
+            dstStore->registerDrvOutput(realisation);
+        }
+    } catch (MissingExperimentalFeature & e) {
+        // Don't fail if the remote doesn't support CA derivations is it might
+        // not be whithin our control to change that, and we might still want
+        // to at least copy the output paths.
+        if (e.missingFeature == "ca-derivations")
+            ignoreException();
+        else
+            throw;
     }
 
     return pathsMap;

From 3b76f8f252c12fbeb49aa2f6f695b4622e9fcc5d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 19 Feb 2021 18:02:26 +0100
Subject: [PATCH 765/882] Ensure that the ca-derivations bit is set when
 copying realisations

This should already hold, but better ensure it for future-proof-nees
---
 src/libstore/store-api.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ac1d8ee2c..db84ec7a2 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -786,6 +786,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
+    settings.requireExperimentalFeature("ca-derivations");
     StorePathSet storePaths;
     std::set<Realisation> realisations;
     for (auto path : paths) {

From c182aac98ab6548c16b6686638591ba5b034026a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Thu, 25 Feb 2021 17:10:45 +0100
Subject: [PATCH 766/882] Apply @edolstra stylistic suggestions

Mostly removing useless comments and adding spaces before `&`

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/globals.hh   | 1 -
 src/libstore/store-api.cc | 6 +++---
 src/libstore/store-api.hh | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 25351f55c..a51d9c2f1 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -45,7 +45,6 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
-/* MakeError(MissingExperimentalFeature, Error); */
 class MissingExperimentalFeature: public Error
 {
 public:
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index db84ec7a2..b7a3f7b11 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -789,19 +789,19 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     settings.requireExperimentalFeature("ca-derivations");
     StorePathSet storePaths;
     std::set<Realisation> realisations;
-    for (auto path : paths) {
+    for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto realisation = std::get_if<Realisation>(&path.raw))
             realisations.insert(*realisation);
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
     try {
-        for (auto& realisation : realisations) {
+        for (auto & realisation : realisations) {
             dstStore->registerDrvOutput(realisation);
         }
     } catch (MissingExperimentalFeature & e) {
         // Don't fail if the remote doesn't support CA derivations is it might
-        // not be whithin our control to change that, and we might still want
+        // not be within our control to change that, and we might still want
         // to at least copy the output paths.
         if (e.missingFeature == "ca-derivations")
             ignoreException();
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 63b26422a..742cd18db 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -752,7 +752,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
    that. Returns a map of what each path was copied to the dstStore
    as. */
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
-    const RealisedPath::Set&,
+    const RealisedPath::Set &,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);

From c43f446f4e3a1a8d91560b6ebbcc7d4fbbbf71c4 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 25 Feb 2021 16:58:27 +0100
Subject: [PATCH 767/882] Make `nix copy` work without the ca-derivations flag

The experimental feature was by mistake required for `nix copy` to work
at oll
---
 src/libstore/store-api.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b7a3f7b11..77c310988 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -786,13 +786,14 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
-    settings.requireExperimentalFeature("ca-derivations");
     StorePathSet storePaths;
     std::set<Realisation> realisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
-        if (auto realisation = std::get_if<Realisation>(&path.raw))
+        if (auto realisation = std::get_if<Realisation>(&path.raw)) {
+            settings.requireExperimentalFeature("ca-derivations");
             realisations.insert(*realisation);
+        }
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
     try {

From 20ea1de77d9210e145d5ebb1dccd34c856149b2c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Feb 2021 12:35:29 +0100
Subject: [PATCH 768/882] Use std::make_unique

---
 src/libstore/build/derivation-goal.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 804a79e4c..33c3aeb6e 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -259,10 +259,8 @@ void DerivationGoal::loadDerivation()
 
     assert(worker.store.isValidPath(drvPath));
 
-    auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath));
-
     /* Get the derivation. */
-    drv = std::unique_ptr<Derivation>(fullDrv);
+    drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
 
     haveDerivation();
 }

From 453c3a603f4e6fa3f8c706e73f9869bc7f76c640 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Feb 2021 14:55:54 +0100
Subject: [PATCH 769/882] nix flake update: Recreate the lock file

This is probably what most people expect it to do. Fixes #3781.

There is a new command 'nix flake lock' that has the old behaviour of
'nix flake update', i.e. it just adds missing lock file entries unless
overriden using --update-input.
---
 src/libutil/args.cc     |  8 ++++++++
 src/libutil/args.hh     |  2 ++
 src/nix/flake-lock.md   | 38 ++++++++++++++++++++++++++++++++++++++
 src/nix/flake-update.md | 37 +++++++++----------------------------
 src/nix/flake.cc        | 34 +++++++++++++++++++++++++++++++++-
 tests/flakes.sh         | 28 ++++++++++++++--------------
 6 files changed, 104 insertions(+), 43 deletions(-)
 create mode 100644 src/nix/flake-lock.md

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 75eb19d28..afed0670f 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -19,6 +19,14 @@ void Args::addFlag(Flag && flag_)
     if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
+void Args::removeFlag(const std::string & longName)
+{
+    auto flag = longFlags.find(longName);
+    assert(flag != longFlags.end());
+    if (flag->second->shortName) shortFlags.erase(flag->second->shortName);
+    longFlags.erase(flag);
+}
+
 void Completions::add(std::string completion, std::string description)
 {
     assert(description.find('\n') == std::string::npos);
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 4721c21df..c08ba8abd 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -140,6 +140,8 @@ public:
 
     void addFlag(Flag && flag);
 
+    void removeFlag(const std::string & longName);
+
     void expectArgs(ExpectedArg && arg)
     {
         expectedArgs.emplace_back(std::move(arg));
diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md
new file mode 100644
index 000000000..2af0ad81e
--- /dev/null
+++ b/src/nix/flake-lock.md
@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Update the `nixpkgs` and `nix` inputs of the flake in the current
+  directory:
+
+  ```console
+  # nix flake lock --update-input nixpkgs --update-input nix
+  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  ```
+
+# Description
+
+This command updates the lock file of a flake (`flake.lock`) so that
+it contains a lock for every flake input specified in
+`flake.nix`. Existing lock file entries are not updated unless
+required by a flag such as `--update-input`.
+
+Note that every command that operates on a flake will also update the
+lock file if needed, and supports the same flags. Therefore,
+
+```console
+# nix flake lock --update-input nixpkgs
+# nix build
+```
+
+is equivalent to:
+
+```console
+# nix build --update-input nixpkgs
+```
+
+Thus, this command is only useful if you want to update the lock file
+separately from any other action such as building.
+
+)""
diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md
index a2ffedd2a..03b50e38e 100644
--- a/src/nix/flake-update.md
+++ b/src/nix/flake-update.md
@@ -2,52 +2,33 @@ R""(
 
 # Examples
 
-* Update the `nixpkgs` and `nix` inputs of the flake in the current
-  directory:
-
-  ```console
-  # nix flake update --update-input nixpkgs --update-input nix
-  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
-  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
-  ```
-
 * Recreate the lock file (i.e. update all inputs) and commit the new
   lock file:
 
   ```console
-  # nix flake update --recreate-lock-file --commit-lock-file
+  # nix flake update
+  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
   …
   warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
   ```
 
 # Description
 
-This command updates the lock file of a flake (`flake.lock`) so that
-it contains a lock for every flake input specified in
-`flake.nix`. Note that every command that operates on a flake will
-also update the lock file if needed, and supports the same
-flags. Therefore,
+This command recreates the lock file of a flake (`flake.lock`), thus
+updating the lock for every mutable input (like `nixpkgs`) to its
+current version. This is equivalent to passing `--recreate-lock-file`
+to any command that operates on a flake. That is,
 
 ```console
-# nix flake update --update-input nixpkgs
+# nix flake update
 # nix build
 ```
 
 is equivalent to:
 
 ```console
-# nix build --update-input nixpkgs
+# nix build --recreate-lock-file
 ```
 
-Thus, this command is only useful if you want to update the lock file
-separately from any other action such as building.
-
-> **Note**
->
-> This command does *not* update locks that are already present unless
-> you explicitly ask for it using `--update-input` or
-> `--recreate-lock-file`. Thus, if the lock file already has locks for
-> every input, then `nix flake update` (without arguments) does
-> nothing.
-
 )""
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index b9cde5d6d..2f0c468a8 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -104,6 +104,14 @@ struct CmdFlakeUpdate : FlakeCommand
         return "update flake lock file";
     }
 
+    CmdFlakeUpdate()
+    {
+        /* Remove flags that don't make sense. */
+        removeFlag("recreate-lock-file");
+        removeFlag("update-input");
+        removeFlag("no-update-lock-file");
+    }
+
     std::string doc() override
     {
         return
@@ -113,7 +121,30 @@ struct CmdFlakeUpdate : FlakeCommand
 
     void run(nix::ref<nix::Store> store) override
     {
-        /* Use --refresh by default for 'nix flake update'. */
+        settings.tarballTtl = 0;
+
+        lockFlags.recreateLockFile = true;
+
+        lockFlake();
+    }
+};
+
+struct CmdFlakeLock : FlakeCommand
+{
+    std::string description() override
+    {
+        return "create missing lock file entries";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "flake-lock.md"
+          ;
+    }
+
+    void run(nix::ref<nix::Store> store) override
+    {
         settings.tarballTtl = 0;
 
         lockFlake();
@@ -1006,6 +1037,7 @@ struct CmdFlake : NixMultiCommand
     CmdFlake()
         : MultiCommand({
                 {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
+                {"lock", []() { return make_ref<CmdFlakeLock>(); }},
                 {"info", []() { return make_ref<CmdFlakeInfo>(); }},
                 {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }},
                 {"check", []() { return make_ref<CmdFlakeCheck>(); }},
diff --git a/tests/flakes.sh b/tests/flakes.sh
index 2b7bcdd68..25ba2ac43 100644
--- a/tests/flakes.sh
+++ b/tests/flakes.sh
@@ -232,7 +232,7 @@ nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2
 nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh
 
 # Updating the flake should not change the lockfile.
-nix flake update $flake2Dir
+nix flake lock $flake2Dir
 [[ -z $(git -C $flake2Dir diff master) ]]
 
 # Now we should be able to build the flake in pure mode.
@@ -354,10 +354,10 @@ nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord
 nix build -o $TEST_ROOT/result flake4#xyzzy
 
 # Test 'nix flake update' and --override-flake.
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ -z $(git -C $flake3Dir diff master) ]]
 
-nix flake update $flake3Dir --recreate-lock-file --override-flake flake2 nixpkgs
+nix flake update $flake3Dir --override-flake flake2 nixpkgs
 [[ ! -z $(git -C $flake3Dir diff master) ]]
 
 # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
@@ -389,7 +389,7 @@ cat > $flake3Dir/flake.nix <<EOF
   };
 }
 EOF
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 git -C $flake3Dir add flake.nix flake.lock
 git -C $flake3Dir commit -m 'Remove packages.xyzzy'
 git -C $flake3Dir checkout master
@@ -547,7 +547,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["foo"]' ]]
 
 cat > $flake3Dir/flake.nix <<EOF
@@ -559,7 +559,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2","flake1"]' ]]
 
 cat > $flake3Dir/flake.nix <<EOF
@@ -571,7 +571,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2"]' ]]
 
 # Test overriding inputs of inputs.
@@ -587,7 +587,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq .nodes.flake1.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
 
 cat > $flake3Dir/flake.nix <<EOF
@@ -600,7 +600,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir --recreate-lock-file
+nix flake update $flake3Dir
 [[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]]
 [[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
 
@@ -658,20 +658,20 @@ nix build -o $TEST_ROOT/result "file://$TEST_ROOT/flake.tar.gz?narHash=sha256-qQ
 
 # Test --override-input.
 git -C $flake3Dir reset --hard
-nix flake update $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
+nix flake lock $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
 [[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake5 ]]
 
-nix flake update $flake3Dir --override-input flake2/flake1 flake1
+nix flake lock $flake3Dir --override-input flake2/flake1 flake1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
 
-nix flake update $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
+nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]]
 
 # Test --update-input.
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]]
 
-nix flake update $flake3Dir --update-input flake2/flake1
+nix flake lock $flake3Dir --update-input flake2/flake1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
 
 # Test 'nix flake list-inputs'.

From 8d322f3c942165f451959229416f51d54ae24c28 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Feb 2021 16:03:39 +0100
Subject: [PATCH 770/882] flake.lock: Update

Flake input changes:

* Updated 'nixpkgs': 'github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31' -> 'github:NixOS/nixpkgs/0e499fde7af3c28d63e9b13636716b86c3162b93'
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 6fe52fbfd..9867e694b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1602702596,
-        "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=",
+        "lastModified": 1614309161,
+        "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ad0d20345219790533ebe06571f82ed6b034db31",
+        "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93",
         "type": "github"
       },
       "original": {

From 14f51880bad5145e73eb150797e757440925913b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Feb 2021 16:29:30 +0100
Subject: [PATCH 771/882] Update src/build-remote/build-remote.cc

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 228aba35a..1be491603 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -288,7 +288,7 @@ connected:
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
                 if (!store->queryRealisation(thisOutputId)) {
-                    notice("Missing output %s", outputName);
+                    debug("missing output %s", outputName);
                     assert(result.builtOutputs.count(thisOutputId));
                     auto newRealisation = result.builtOutputs.at(thisOutputId);
                     missingRealisations.insert(newRealisation);

From 17c98e03eac45b3c298567e8a1c04e3d4c4aa0d2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Feb 2021 16:29:37 +0100
Subject: [PATCH 772/882] Update src/build-remote/build-remote.cc

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 1be491603..7f3636f6b 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -311,7 +311,7 @@ connected:
                     localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
         }
-        // XXX: Should e done as part of `copyPaths`
+        // XXX: Should be done as part of `copyPaths`
         for (auto & realisation : missingRealisations) {
             // Should hold, because if the feature isn't enabled the set
             // of missing realisations should be empty

From 076d2b04da72607b67e581572a31db2a220589ed Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Feb 2021 16:30:12 +0100
Subject: [PATCH 773/882] Update src/libstore/build/derivation-goal.cc

---
 src/libstore/build/derivation-goal.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 6052b625d..a5622f990 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1164,7 +1164,6 @@ HookReply DerivationGoal::tryBuildHook()
             // XXX: Does this include known CA outputs?
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
-            /* missingPaths.insert(status.known->path); */
         }
         worker_proto::write(worker.store, hook->sink, missingOutputs);
     }

From f54976d77bd144535e9b4844dbdb6bc52eac11fd Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 26 Feb 2021 16:34:33 +0100
Subject: [PATCH 774/882] Simplify the case where the drv is a purely
 input-addressed one

---
 src/build-remote/build-remote.cc   |  2 +-
 src/libstore/build/entry-points.cc |  2 +-
 src/libstore/derivations.cc        | 11 +++++++++++
 src/libstore/derivations.hh        |  5 +++++
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 7f3636f6b..736b81542 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -283,7 +283,7 @@ connected:
 
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 99b3fa070..3a05a022c 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -69,7 +69,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
                     outputId,
                     Realisation{ outputId, *staticOutput.second}
                     );
-        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
             auto realisation = this->queryRealisation(outputId);
             if (realisation)
                 result.builtOutputs.insert_or_assign(
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 6d0742b4f..fe98182bb 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -57,6 +57,17 @@ bool derivationIsFixed(DerivationType dt) {
     assert(false);
 }
 
+bool derivationHasKnownOutputPaths(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return true;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return false;
+    case DerivationType::DeferredInputAddressed: return false;
+    };
+    assert(false);
+}
+
+
 bool derivationIsImpure(DerivationType dt) {
     switch (dt) {
     case DerivationType::InputAddressed: return false;
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 4e5985fab..061d70f69 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -94,6 +94,11 @@ bool derivationIsFixed(DerivationType);
    derivation is controlled separately. Never true for non-CA derivations. */
 bool derivationIsImpure(DerivationType);
 
+/* Does the derivation knows its own output paths?
+ * Only true when there's no floating-ca derivation involved in the closure.
+ */
+bool derivationHasKnownOutputPaths(DerivationType);
+
 struct BasicDerivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */

From 05cc5a858717c092e1835e2b0fec4c4b1a7fc97e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 23 Feb 2021 06:26:35 +0000
Subject: [PATCH 775/882] Copy {,local-}derivation-goal.{cc,h}

Doing this prior to splitting, so we get better diff with default
options (e.g. on GitHub).
---
 src/libstore/build/local-derivation-goal.cc | 3902 +++++++++++++++++++
 src/libstore/build/local-derivation-goal.hh |  373 ++
 2 files changed, 4275 insertions(+)
 create mode 100644 src/libstore/build/local-derivation-goal.cc
 create mode 100644 src/libstore/build/local-derivation-goal.hh

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
new file mode 100644
index 000000000..924c69fb7
--- /dev/null
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -0,0 +1,3902 @@
+#include "derivation-goal.hh"
+#include "hook-instance.hh"
+#include "worker.hh"
+#include "builtins.hh"
+#include "builtins/buildenv.hh"
+#include "references.hh"
+#include "finally.hh"
+#include "util.hh"
+#include "archive.hh"
+#include "json.hh"
+#include "compression.hh"
+#include "daemon.hh"
+#include "worker-protocol.hh"
+#include "topo-sort.hh"
+#include "callback.hh"
+
+#include <regex>
+#include <queue>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netdb.h>
+#include <fcntl.h>
+#include <termios.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+
+#if HAVE_STATVFS
+#include <sys/statvfs.h>
+#endif
+
+/* Includes required for chroot support. */
+#if __linux__
+#include <sys/socket.h>
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <netinet/ip.h>
+#include <sys/personality.h>
+#include <sys/mman.h>
+#include <sched.h>
+#include <sys/param.h>
+#include <sys/mount.h>
+#include <sys/syscall.h>
+#if HAVE_SECCOMP
+#include <seccomp.h>
+#endif
+#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
+#endif
+
+#if __APPLE__
+#include <spawn.h>
+#include <sys/sysctl.h>
+#endif
+
+#include <pwd.h>
+#include <grp.h>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+void handleDiffHook(
+    uid_t uid, uid_t gid,
+    const Path & tryA, const Path & tryB,
+    const Path & drvPath, const Path & tmpDir)
+{
+    auto diffHook = settings.diffHook;
+    if (diffHook != "" && settings.runDiffHook) {
+        try {
+            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
+            diffHookOptions.searchPath = true;
+            diffHookOptions.uid = uid;
+            diffHookOptions.gid = gid;
+            diffHookOptions.chdir = "/";
+
+            auto diffRes = runProgram(diffHookOptions);
+            if (!statusOk(diffRes.first))
+                throw ExecError(diffRes.first,
+                    "diff-hook program '%1%' %2%",
+                    diffHook,
+                    statusToString(diffRes.first));
+
+            if (diffRes.second != "")
+                printError(chomp(diffRes.second));
+        } catch (Error & error) {
+            ErrorInfo ei = error.info();
+            // FIXME: wrap errors.
+            ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str());
+            logError(ei);
+        }
+    }
+}
+
+const Path DerivationGoal::homeDir = "/homeless-shelter";
+
+DerivationGoal::DerivationGoal(const StorePath & drvPath,
+    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
+    : Goal(worker)
+    , useDerivation(true)
+    , drvPath(drvPath)
+    , wantedOutputs(wantedOutputs)
+    , buildMode(buildMode)
+{
+    state = &DerivationGoal::getDerivation;
+    name = fmt(
+        "building of '%s' from .drv file",
+        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
+    trace("created");
+
+    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
+    worker.updateProgress();
+}
+
+
+DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
+    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
+    : Goal(worker)
+    , useDerivation(false)
+    , drvPath(drvPath)
+    , wantedOutputs(wantedOutputs)
+    , buildMode(buildMode)
+{
+    this->drv = std::make_unique<Derivation>(drv);
+
+    state = &DerivationGoal::haveDerivation;
+    name = fmt(
+        "building of '%s' from in-memory derivation",
+        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
+    trace("created");
+
+    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
+    worker.updateProgress();
+
+    /* Prevent the .chroot directory from being
+       garbage-collected. (See isActiveTempFile() in gc.cc.) */
+    worker.store.addTempRoot(this->drvPath);
+}
+
+
+DerivationGoal::~DerivationGoal()
+{
+    /* Careful: we should never ever throw an exception from a
+       destructor. */
+    try { killChild(); } catch (...) { ignoreException(); }
+    try { stopDaemon(); } catch (...) { ignoreException(); }
+    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
+    try { closeLogFile(); } catch (...) { ignoreException(); }
+}
+
+
+string DerivationGoal::key()
+{
+    /* Ensure that derivations get built in order of their name,
+       i.e. a derivation named "aardvark" always comes before
+       "baboon". And substitution goals always happen before
+       derivation goals (due to "b$"). */
+    return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
+}
+
+
+inline bool DerivationGoal::needsHashRewrite()
+{
+#if __linux__
+    return !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    return true;
+#endif
+}
+
+
+void DerivationGoal::killChild()
+{
+    if (pid != -1) {
+        worker.childTerminated(this);
+
+        if (buildUser) {
+            /* If we're using a build user, then there is a tricky
+               race condition: if we kill the build user before the
+               child has done its setuid() to the build user uid, then
+               it won't be killed, and we'll potentially lock up in
+               pid.wait().  So also send a conventional kill to the
+               child. */
+            ::kill(-pid, SIGKILL); /* ignore the result */
+            buildUser->kill();
+            pid.wait();
+        } else
+            pid.kill();
+
+        assert(pid == -1);
+    }
+
+    hook.reset();
+}
+
+
+void DerivationGoal::timedOut(Error && ex)
+{
+    killChild();
+    done(BuildResult::TimedOut, ex);
+}
+
+
+void DerivationGoal::work()
+{
+    (this->*state)();
+}
+
+
+void DerivationGoal::addWantedOutputs(const StringSet & outputs)
+{
+    /* If we already want all outputs, there is nothing to do. */
+    if (wantedOutputs.empty()) return;
+
+    if (outputs.empty()) {
+        wantedOutputs.clear();
+        needRestart = true;
+    } else
+        for (auto & i : outputs)
+            if (wantedOutputs.insert(i).second)
+                needRestart = true;
+}
+
+
+void DerivationGoal::getDerivation()
+{
+    trace("init");
+
+    /* The first thing to do is to make sure that the derivation
+       exists.  If it doesn't, it may be created through a
+       substitute. */
+    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
+        loadDerivation();
+        return;
+    }
+
+    addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath)));
+
+    state = &DerivationGoal::loadDerivation;
+}
+
+
+void DerivationGoal::loadDerivation()
+{
+    trace("loading derivation");
+
+    if (nrFailed != 0) {
+        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
+        return;
+    }
+
+    /* `drvPath' should already be a root, but let's be on the safe
+       side: if the user forgot to make it a root, we wouldn't want
+       things being garbage collected while we're busy. */
+    worker.store.addTempRoot(drvPath);
+
+    assert(worker.store.isValidPath(drvPath));
+
+    /* Get the derivation. */
+    drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
+
+    haveDerivation();
+}
+
+
+void DerivationGoal::haveDerivation()
+{
+    trace("have derivation");
+
+    if (drv->type() == DerivationType::CAFloating)
+        settings.requireExperimentalFeature("ca-derivations");
+
+    retrySubstitution = false;
+
+    for (auto & i : drv->outputsAndOptPaths(worker.store))
+        if (i.second.second)
+            worker.store.addTempRoot(*i.second.second);
+
+    auto outputHashes = staticOutputHashes(worker.store, *drv);
+    for (auto &[outputName, outputHash] : outputHashes)
+      initialOutputs.insert({
+            outputName,
+            InitialOutput{
+                .wanted = true, // Will be refined later
+                .outputHash = outputHash
+            }
+          });
+
+    /* Check what outputs paths are not already valid. */
+    checkPathValidity();
+    bool allValid = true;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid()) {
+            allValid = false;
+            break;
+        }
+    }
+
+    /* If they are all valid, then we're done. */
+    if (allValid && buildMode == bmNormal) {
+        done(BuildResult::AlreadyValid);
+        return;
+    }
+
+    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+
+
+    /* We are first going to try to create the invalid output paths
+       through substitutes.  If that doesn't work, we'll build
+       them. */
+    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
+        for (auto & [_, status] : initialOutputs) {
+            if (!status.wanted) continue;
+            if (!status.known) {
+                warn("do not know how to query for unknown floating content-addressed derivation output yet");
+                /* Nothing to wait for; tail call */
+                return DerivationGoal::gaveUpOnSubstitution();
+            }
+            addWaitee(upcast_goal(worker.makeSubstitutionGoal(
+                status.known->path,
+                buildMode == bmRepair ? Repair : NoRepair,
+                getDerivationCA(*drv))));
+        }
+
+    if (waitees.empty()) /* to prevent hang (no wake-up event) */
+        outputsSubstitutionTried();
+    else
+        state = &DerivationGoal::outputsSubstitutionTried;
+}
+
+
+void DerivationGoal::outputsSubstitutionTried()
+{
+    trace("all outputs substituted (maybe)");
+
+    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
+        done(BuildResult::TransientFailure,
+            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
+                worker.store.printStorePath(drvPath)));
+        return;
+    }
+
+    /*  If the substitutes form an incomplete closure, then we should
+        build the dependencies of this derivation, but after that, we
+        can still use the substitutes for this derivation itself.
+
+        If the nrIncompleteClosure != nrFailed, we have another issue as well.
+        In particular, it may be the case that the hole in the closure is
+        an output of the current derivation, which causes a loop if retried.
+     */
+    if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true;
+
+    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
+
+    if (needRestart) {
+        needRestart = false;
+        haveDerivation();
+        return;
+    }
+
+    checkPathValidity();
+    size_t nrInvalid = 0;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid())
+            nrInvalid++;
+    }
+
+    if (buildMode == bmNormal && nrInvalid == 0) {
+        done(BuildResult::Substituted);
+        return;
+    }
+    if (buildMode == bmRepair && nrInvalid == 0) {
+        repairClosure();
+        return;
+    }
+    if (buildMode == bmCheck && nrInvalid > 0)
+        throw Error("some outputs of '%s' are not valid, so checking is not possible",
+            worker.store.printStorePath(drvPath));
+
+    /* Nothing to wait for; tail call */
+    gaveUpOnSubstitution();
+}
+
+/* At least one of the output paths could not be
+   produced using a substitute.  So we have to build instead. */
+void DerivationGoal::gaveUpOnSubstitution()
+{
+    /* Make sure checkPathValidity() from now on checks all
+       outputs. */
+    wantedOutputs.clear();
+
+    /* The inputs must be built before we can build this goal. */
+    if (useDerivation)
+        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
+            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+
+    for (auto & i : drv->inputSrcs) {
+        if (worker.store.isValidPath(i)) continue;
+        if (!settings.useSubstitutes)
+            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
+                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
+        addWaitee(upcast_goal(worker.makeSubstitutionGoal(i)));
+    }
+
+    if (waitees.empty()) /* to prevent hang (no wake-up event) */
+        inputsRealised();
+    else
+        state = &DerivationGoal::inputsRealised;
+}
+
+
+void DerivationGoal::repairClosure()
+{
+    /* If we're repairing, we now know that our own outputs are valid.
+       Now check whether the other paths in the outputs closure are
+       good.  If not, then start derivation goals for the derivations
+       that produced those outputs. */
+
+    /* Get the output closure. */
+    auto outputs = queryDerivationOutputMap();
+    StorePathSet outputClosure;
+    for (auto & i : outputs) {
+        if (!wantOutput(i.first, wantedOutputs)) continue;
+        worker.store.computeFSClosure(i.second, outputClosure);
+    }
+
+    /* Filter out our own outputs (which we have already checked). */
+    for (auto & i : outputs)
+        outputClosure.erase(i.second);
+
+    /* Get all dependencies of this derivation so that we know which
+       derivation is responsible for which path in the output
+       closure. */
+    StorePathSet inputClosure;
+    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
+    std::map<StorePath, StorePath> outputsToDrv;
+    for (auto & i : inputClosure)
+        if (i.isDerivation()) {
+            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
+            for (auto & j : depOutputs)
+                if (j.second)
+                    outputsToDrv.insert_or_assign(*j.second, i);
+        }
+
+    /* Check each path (slow!). */
+    for (auto & i : outputClosure) {
+        if (worker.pathContentsGood(i)) continue;
+        printError(
+            "found corrupted or missing path '%s' in the output closure of '%s'",
+            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
+        auto drvPath2 = outputsToDrv.find(i);
+        if (drvPath2 == outputsToDrv.end())
+            addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
+        else
+            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
+    }
+
+    if (waitees.empty()) {
+        done(BuildResult::AlreadyValid);
+        return;
+    }
+
+    state = &DerivationGoal::closureRepaired;
+}
+
+
+void DerivationGoal::closureRepaired()
+{
+    trace("closure repaired");
+    if (nrFailed > 0)
+        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
+            worker.store.printStorePath(drvPath));
+    done(BuildResult::AlreadyValid);
+}
+
+
+void DerivationGoal::inputsRealised()
+{
+    trace("all inputs realised");
+
+    if (nrFailed != 0) {
+        if (!useDerivation)
+            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
+        done(BuildResult::DependencyFailed, Error(
+                "%s dependencies of derivation '%s' failed to build",
+                nrFailed, worker.store.printStorePath(drvPath)));
+        return;
+    }
+
+    if (retrySubstitution) {
+        haveDerivation();
+        return;
+    }
+
+    /* Gather information necessary for computing the closure and/or
+       running the build hook. */
+
+    /* Determine the full set of input paths. */
+
+    /* First, the input derivations. */
+    if (useDerivation) {
+        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
+
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+            ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
+            || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
+            /* We are be able to resolve this derivation based on the
+               now-known results of dependencies. If so, we become a stub goal
+               aliasing that resolved derivation goal */
+            std::optional attempt = fullDrv.tryResolve(worker.store);
+            assert(attempt);
+            Derivation drvResolved { *std::move(attempt) };
+
+            auto pathResolved = writeDerivation(worker.store, drvResolved);
+            resolvedDrv = drvResolved;
+
+            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
+                worker.store.printStorePath(drvPath),
+                worker.store.printStorePath(pathResolved));
+            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
+                Logger::Fields {
+                       worker.store.printStorePath(drvPath),
+                       worker.store.printStorePath(pathResolved),
+                   });
+
+            auto resolvedGoal = worker.makeDerivationGoal(
+                pathResolved, wantedOutputs, buildMode);
+            addWaitee(resolvedGoal);
+
+            state = &DerivationGoal::resolvedFinished;
+            return;
+        }
+
+        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
+            /* Add the relevant output closures of the input derivation
+               `i' as input paths.  Only add the closures of output paths
+               that are specified as inputs. */
+            assert(worker.store.isValidPath(drvPath));
+            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
+            for (auto & j : wantedDepOutputs) {
+                if (outputs.count(j) > 0) {
+                    auto optRealizedInput = outputs.at(j);
+                    if (!optRealizedInput)
+                        throw Error(
+                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
+                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
+                } else
+                    throw Error(
+                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
+                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+            }
+        }
+    }
+
+    /* Second, the input sources. */
+    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
+
+    debug("added input paths %s", worker.store.showPaths(inputPaths));
+
+    /* What type of derivation are we building? */
+    derivationType = drv->type();
+
+    /* Don't repeat fixed-output derivations since they're already
+       verified by their output hash.*/
+    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
+
+    /* Okay, try to build.  Note that here we don't wait for a build
+       slot to become available, since we don't need one if there is a
+       build hook. */
+    state = &DerivationGoal::tryToBuild;
+    worker.wakeUp(shared_from_this());
+
+    result = BuildResult();
+}
+
+void DerivationGoal::started() {
+    auto msg = fmt(
+        buildMode == bmRepair ? "repairing outputs of '%s'" :
+        buildMode == bmCheck ? "checking outputs of '%s'" :
+        nrRounds > 1 ? "building '%s' (round %d/%d)" :
+        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
+    fmt("building '%s'", worker.store.printStorePath(drvPath));
+    if (hook) msg += fmt(" on '%s'", machineName);
+    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
+        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
+    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
+    worker.updateProgress();
+}
+
+void DerivationGoal::tryToBuild()
+{
+    trace("trying to build");
+
+    /* Obtain locks on all output paths, if the paths are known a priori.
+
+       The locks are automatically released when we exit this function or Nix
+       crashes.  If we can't acquire the lock, then continue; hopefully some
+       other goal can start a build, and if not, the main loop will sleep a few
+       seconds and then retry this goal. */
+    PathSet lockFiles;
+    /* FIXME: Should lock something like the drv itself so we don't build same
+       CA drv concurrently */
+    if (dynamic_cast<LocalStore *>(&worker.store))
+        /* If we aren't a local store, we might need to use the local store as
+           a build remote, but that would cause a deadlock. */
+        /* FIXME: Make it so we can use ourselves as a build remote even if we
+           are the local store (separate locking for building vs scheduling? */
+        /* FIXME: find some way to lock for scheduling for the other stores so
+           a forking daemon with --store still won't farm out redundant builds.
+           */
+        for (auto & i : drv->outputsAndOptPaths(worker.store))
+            if (i.second.second)
+                lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+
+    if (!outputLocks.lockPaths(lockFiles, "", false)) {
+        if (!actLock)
+            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
+        worker.waitForAWhile(shared_from_this());
+        return;
+    }
+
+    actLock.reset();
+
+    /* Now check again whether the outputs are valid.  This is because
+       another process may have started building in parallel.  After
+       it has finished and released the locks, we can (and should)
+       reuse its results.  (Strictly speaking the first check can be
+       omitted, but that would be less efficient.)  Note that since we
+       now hold the locks on the output paths, no other process can
+       build this derivation, so no further checks are necessary. */
+    checkPathValidity();
+    bool allValid = true;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid()) {
+            allValid = false;
+            break;
+        }
+    }
+    if (buildMode != bmCheck && allValid) {
+        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
+        outputLocks.setDeletion(true);
+        done(BuildResult::AlreadyValid);
+        return;
+    }
+
+    /* If any of the outputs already exist but are not valid, delete
+       them. */
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.known || status.known->isValid()) continue;
+        auto storePath = status.known->path;
+        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
+        deletePath(worker.store.Store::toRealPath(storePath));
+    }
+
+    /* Don't do a remote build if the derivation has the attribute
+       `preferLocalBuild' set.  Also, check and repair modes are only
+       supported for local builds. */
+    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+
+    if (!buildLocally) {
+        switch (tryBuildHook()) {
+            case rpAccept:
+                /* Yes, it has started doing so.  Wait until we get
+                   EOF from the hook. */
+                actLock.reset();
+                result.startTime = time(0); // inexact
+                state = &DerivationGoal::buildDone;
+                started();
+                return;
+            case rpPostpone:
+                /* Not now; wait until at least one child finishes or
+                   the wake-up timeout expires. */
+                if (!actLock)
+                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
+                worker.waitForAWhile(shared_from_this());
+                outputLocks.unlock();
+                return;
+            case rpDecline:
+                /* We should do it ourselves. */
+                break;
+        }
+    }
+
+    actLock.reset();
+
+    state = &DerivationGoal::tryLocalBuild;
+    worker.wakeUp(shared_from_this());
+}
+
+void DerivationGoal::tryLocalBuild() {
+    /* Make sure that we are allowed to start a build. */
+    if (!dynamic_cast<LocalStore *>(&worker.store)) {
+        throw Error(
+            "unable to build with a primary store that isn't a local store; "
+            "either pass a different '--store' or enable remote builds."
+            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+    }
+    unsigned int curBuilds = worker.getNrLocalBuilds();
+    if (curBuilds >= settings.maxBuildJobs) {
+        worker.waitForBuildSlot(shared_from_this());
+        outputLocks.unlock();
+        return;
+    }
+
+    /* If `build-users-group' is not empty, then we have to build as
+       one of the members of that group. */
+    if (settings.buildUsersGroup != "" && getuid() == 0) {
+#if defined(__linux__) || defined(__APPLE__)
+        if (!buildUser) buildUser = std::make_unique<UserLock>();
+
+        if (buildUser->findFreeUser()) {
+            /* Make sure that no other processes are executing under this
+               uid. */
+            buildUser->kill();
+        } else {
+            if (!actLock)
+                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
+            worker.waitForAWhile(shared_from_this());
+            return;
+        }
+#else
+        /* Don't know how to block the creation of setuid/setgid
+           binaries on this platform. */
+        throw Error("build users are not supported on this platform for security reasons");
+#endif
+    }
+
+    actLock.reset();
+
+    try {
+
+        /* Okay, we have to build. */
+        startBuilder();
+
+    } catch (BuildError & e) {
+        outputLocks.unlock();
+        buildUser.reset();
+        worker.permanentFailure = true;
+        done(BuildResult::InputRejected, e);
+        return;
+    }
+
+    /* This state will be reached when we get EOF on the child's
+       log pipe. */
+    state = &DerivationGoal::buildDone;
+
+    started();
+}
+
+
+static void chmod_(const Path & path, mode_t mode)
+{
+    if (chmod(path.c_str(), mode) == -1)
+        throw SysError("setting permissions on '%s'", path);
+}
+
+
+/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
+   it's a directory and we're not root (to be able to update the
+   directory's parent link ".."). */
+static void movePath(const Path & src, const Path & dst)
+{
+    auto st = lstat(src);
+
+    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+    if (changePerm)
+        chmod_(src, st.st_mode | S_IWUSR);
+
+    if (rename(src.c_str(), dst.c_str()))
+        throw SysError("renaming '%1%' to '%2%'", src, dst);
+
+    if (changePerm)
+        chmod_(dst, st.st_mode);
+}
+
+
+void replaceValidPath(const Path & storePath, const Path & tmpPath)
+{
+    /* We can't atomically replace storePath (the original) with
+       tmpPath (the replacement), so we have to move it out of the
+       way first.  We'd better not be interrupted here, because if
+       we're repairing (say) Glibc, we end up with a broken system. */
+    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
+    if (pathExists(storePath))
+        movePath(storePath, oldPath);
+
+    try {
+        movePath(tmpPath, storePath);
+    } catch (...) {
+        try {
+            // attempt to recover
+            movePath(oldPath, storePath);
+        } catch (...) {
+            ignoreException();
+        }
+        throw;
+    }
+
+    deletePath(oldPath);
+}
+
+
+MakeError(NotDeterministic, BuildError);
+
+
+void DerivationGoal::buildDone()
+{
+    trace("build done");
+
+    /* Release the build user at the end of this function. We don't do
+       it right away because we don't want another build grabbing this
+       uid and then messing around with our output. */
+    Finally releaseBuildUser([&]() { buildUser.reset(); });
+
+    sandboxMountNamespace = -1;
+
+    /* Since we got an EOF on the logger pipe, the builder is presumed
+       to have terminated.  In fact, the builder could also have
+       simply have closed its end of the pipe, so just to be sure,
+       kill it. */
+    int status = hook ? hook->pid.kill() : pid.kill();
+
+    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
+
+    result.timesBuilt++;
+    result.stopTime = time(0);
+
+    /* So the child is gone now. */
+    worker.childTerminated(this);
+
+    /* Close the read side of the logger pipe. */
+    if (hook) {
+        hook->builderOut.readSide = -1;
+        hook->fromHook.readSide = -1;
+    } else
+        builderOut.readSide = -1;
+
+    /* Close the log file. */
+    closeLogFile();
+
+    /* When running under a build user, make sure that all processes
+       running under that uid are gone.  This is to prevent a
+       malicious user from leaving behind a process that keeps files
+       open and modifies them after they have been chown'ed to
+       root. */
+    if (buildUser) buildUser->kill();
+
+    /* Terminate the recursive Nix daemon. */
+    stopDaemon();
+
+    bool diskFull = false;
+
+    try {
+
+        /* Check the exit status. */
+        if (!statusOk(status)) {
+
+            /* Heuristically check whether the build failure may have
+               been caused by a disk full condition.  We have no way
+               of knowing whether the build actually got an ENOSPC.
+               So instead, check if the disk is (nearly) full now.  If
+               so, we don't mark this build as a permanent failure. */
+#if HAVE_STATVFS
+            if (auto localStore = dynamic_cast<LocalStore *>(&worker.store)) {
+                uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
+                struct statvfs st;
+                if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 &&
+                    (uint64_t) st.f_bavail * st.f_bsize < required)
+                    diskFull = true;
+                if (statvfs(tmpDir.c_str(), &st) == 0 &&
+                    (uint64_t) st.f_bavail * st.f_bsize < required)
+                    diskFull = true;
+            }
+#endif
+
+            deleteTmpDir(false);
+
+            /* Move paths out of the chroot for easier debugging of
+               build failures. */
+            if (useChroot && buildMode == bmNormal)
+                for (auto & [_, status] : initialOutputs) {
+                    if (!status.known) continue;
+                    if (buildMode != bmCheck && status.known->isValid()) continue;
+                    auto p = worker.store.printStorePath(status.known->path);
+                    if (pathExists(chrootRootDir + p))
+                        rename((chrootRootDir + p).c_str(), p.c_str());
+                }
+
+            auto msg = fmt("builder for '%s' %s",
+                yellowtxt(worker.store.printStorePath(drvPath)),
+                statusToString(status));
+
+            if (!logger->isVerbose() && !logTail.empty()) {
+                msg += fmt(";\nlast %d log lines:\n", logTail.size());
+                for (auto & line : logTail) {
+                    msg += "> ";
+                    msg += line;
+                    msg += "\n";
+                }
+                msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.",
+                    worker.store.printStorePath(drvPath));
+            }
+
+            if (diskFull)
+                msg += "\nnote: build failure may have been caused by lack of free disk space";
+
+            throw BuildError(msg);
+        }
+
+        /* Compute the FS closure of the outputs and register them as
+           being valid. */
+        registerOutputs();
+
+        if (settings.postBuildHook != "") {
+            Activity act(*logger, lvlInfo, actPostBuildHook,
+                fmt("running post-build-hook '%s'", settings.postBuildHook),
+                Logger::Fields{worker.store.printStorePath(drvPath)});
+            PushActivity pact(act.id);
+            StorePathSet outputPaths;
+            for (auto i : drv->outputs) {
+                outputPaths.insert(finalOutputs.at(i.first));
+            }
+            std::map<std::string, std::string> hookEnvironment = getEnv();
+
+            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
+
+            RunOptions opts(settings.postBuildHook, {});
+            opts.environment = hookEnvironment;
+
+            struct LogSink : Sink {
+                Activity & act;
+                std::string currentLine;
+
+                LogSink(Activity & act) : act(act) { }
+
+                void operator() (std::string_view data) override {
+                    for (auto c : data) {
+                        if (c == '\n') {
+                            flushLine();
+                        } else {
+                            currentLine += c;
+                        }
+                    }
+                }
+
+                void flushLine() {
+                    act.result(resPostBuildLogLine, currentLine);
+                    currentLine.clear();
+                }
+
+                ~LogSink() {
+                    if (currentLine != "") {
+                        currentLine += '\n';
+                        flushLine();
+                    }
+                }
+            };
+            LogSink sink(act);
+
+            opts.standardOut = &sink;
+            opts.mergeStderrToStdout = true;
+            runProgram2(opts);
+        }
+
+        if (buildMode == bmCheck) {
+            deleteTmpDir(true);
+            done(BuildResult::Built);
+            return;
+        }
+
+        /* Delete unused redirected outputs (when doing hash rewriting). */
+        for (auto & i : redirectedOutputs)
+            deletePath(worker.store.Store::toRealPath(i.second));
+
+        /* Delete the chroot (if we were using one). */
+        autoDelChroot.reset(); /* this runs the destructor */
+
+        deleteTmpDir(true);
+
+        /* Repeat the build if necessary. */
+        if (curRound++ < nrRounds) {
+            outputLocks.unlock();
+            state = &DerivationGoal::tryToBuild;
+            worker.wakeUp(shared_from_this());
+            return;
+        }
+
+        /* It is now safe to delete the lock files, since all future
+           lockers will see that the output paths are valid; they will
+           not create new lock files with the same names as the old
+           (unlinked) lock files. */
+        outputLocks.setDeletion(true);
+        outputLocks.unlock();
+
+    } catch (BuildError & e) {
+        outputLocks.unlock();
+
+        BuildResult::Status st = BuildResult::MiscFailure;
+
+        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
+            st = BuildResult::TimedOut;
+
+        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
+        }
+
+        else {
+            st =
+                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
+                statusOk(status) ? BuildResult::OutputRejected :
+                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
+                BuildResult::PermanentFailure;
+        }
+
+        done(st, e);
+        return;
+    }
+
+    done(BuildResult::Built);
+}
+
+void DerivationGoal::resolvedFinished() {
+    assert(resolvedDrv);
+
+    auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
+
+    // `wantedOutputs` might be empty, which means “all the outputs”
+    auto realWantedOutputs = wantedOutputs;
+    if (realWantedOutputs.empty())
+        realWantedOutputs = resolvedDrv->outputNames();
+
+    for (auto & wantedOutput : realWantedOutputs) {
+        assert(initialOutputs.count(wantedOutput) != 0);
+        assert(resolvedHashes.count(wantedOutput) != 0);
+        auto realisation = worker.store.queryRealisation(
+                DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
+        );
+        // We've just built it, but maybe the build failed, in which case the
+        // realisation won't be there
+        if (realisation) {
+            auto newRealisation = *realisation;
+            newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
+            worker.store.registerDrvOutput(newRealisation);
+        } else {
+            // If we don't have a realisation, then it must mean that something
+            // failed when building the resolved drv
+            assert(!result.success());
+        }
+    }
+
+    // This is potentially a bit fishy in terms of error reporting. Not sure
+    // how to do it in a cleaner way
+    amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
+}
+
+HookReply DerivationGoal::tryBuildHook()
+{
+    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
+
+    if (!worker.hook)
+        worker.hook = std::make_unique<HookInstance>();
+
+    try {
+
+        /* Send the request to the hook. */
+        worker.hook->sink
+            << "try"
+            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
+            << drv->platform
+            << worker.store.printStorePath(drvPath)
+            << parsedDrv->getRequiredSystemFeatures();
+        worker.hook->sink.flush();
+
+        /* Read the first line of input, which should be a word indicating
+           whether the hook wishes to perform the build. */
+        string reply;
+        while (true) {
+            auto s = [&]() {
+                try {
+                    return readLine(worker.hook->fromHook.readSide.get());
+                } catch (Error & e) {
+                    e.addTrace({}, "while reading the response from the build hook");
+                    throw e;
+                }
+            }();
+            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
+                ;
+            else if (string(s, 0, 2) == "# ") {
+                reply = string(s, 2);
+                break;
+            }
+            else {
+                s += "\n";
+                writeToStderr(s);
+            }
+        }
+
+        debug("hook reply is '%1%'", reply);
+
+        if (reply == "decline")
+            return rpDecline;
+        else if (reply == "decline-permanently") {
+            worker.tryBuildHook = false;
+            worker.hook = 0;
+            return rpDecline;
+        }
+        else if (reply == "postpone")
+            return rpPostpone;
+        else if (reply != "accept")
+            throw Error("bad hook reply '%s'", reply);
+
+    } catch (SysError & e) {
+        if (e.errNo == EPIPE) {
+            printError(
+                "build hook died unexpectedly: %s",
+                chomp(drainFD(worker.hook->fromHook.readSide.get())));
+            worker.hook = 0;
+            return rpDecline;
+        } else
+            throw;
+    }
+
+    hook = std::move(worker.hook);
+
+    try {
+        machineName = readLine(hook->fromHook.readSide.get());
+    } catch (Error & e) {
+        e.addTrace({}, "while reading the machine name from the build hook");
+        throw e;
+    }
+
+    /* Tell the hook all the inputs that have to be copied to the
+       remote system. */
+    worker_proto::write(worker.store, hook->sink, inputPaths);
+
+    /* Tell the hooks the missing outputs that have to be copied back
+       from the remote system. */
+    {
+        StringSet missingOutputs;
+        for (auto & [outputName, status] : initialOutputs) {
+            // XXX: Does this include known CA outputs?
+            if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
+            missingOutputs.insert(outputName);
+        }
+        worker_proto::write(worker.store, hook->sink, missingOutputs);
+    }
+
+    hook->sink = FdSink();
+    hook->toHook.writeSide = -1;
+
+    /* Create the log file and pipe. */
+    Path logFile = openLogFile();
+
+    set<int> fds;
+    fds.insert(hook->fromHook.readSide.get());
+    fds.insert(hook->builderOut.readSide.get());
+    worker.childStarted(shared_from_this(), fds, false, false);
+
+    return rpAccept;
+}
+
+
+int childEntry(void * arg)
+{
+    ((DerivationGoal *) arg)->runChild();
+    return 1;
+}
+
+
+StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
+{
+    StorePathSet paths;
+
+    for (auto & storePath : storePaths) {
+        if (!inputPaths.count(storePath))
+            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
+
+        worker.store.computeFSClosure({storePath}, paths);
+    }
+
+    /* If there are derivations in the graph, then include their
+       outputs as well.  This is useful if you want to do things
+       like passing all build-time dependencies of some path to a
+       derivation that builds a NixOS DVD image. */
+    auto paths2 = paths;
+
+    for (auto & j : paths2) {
+        if (j.isDerivation()) {
+            Derivation drv = worker.store.derivationFromPath(j);
+            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
+                if (!k.second.second)
+                    /* FIXME: I am confused why we are calling
+                       `computeFSClosure` on the output path, rather than
+                       derivation itself. That doesn't seem right to me, so I
+                       won't try to implemented this for CA derivations. */
+                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
+                worker.store.computeFSClosure(*k.second.second, paths);
+            }
+        }
+    }
+
+    return paths;
+}
+
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    std::call_once(dns_resolve_flag, []() {
+        struct addrinfo *res = NULL;
+
+        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
+            if (res) freeaddrinfo(res);
+        }
+    });
+}
+
+
+void linkOrCopy(const Path & from, const Path & to)
+{
+    if (link(from.c_str(), to.c_str()) == -1) {
+        /* Hard-linking fails if we exceed the maximum link count on a
+           file (e.g. 32000 of ext3), which is quite possible after a
+           'nix-store --optimise'. FIXME: actually, why don't we just
+           bind-mount in this case?
+
+           It can also fail with EPERM in BeegFS v7 and earlier versions
+           which don't allow hard-links to other directories */
+        if (errno != EMLINK && errno != EPERM)
+            throw SysError("linking '%s' to '%s'", to, from);
+        copyPath(from, to);
+    }
+}
+
+
+void DerivationGoal::startBuilder()
+{
+    /* Right platform? */
+    if (!parsedDrv->canBuildLocally(worker.store))
+        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
+            drv->platform,
+            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
+            worker.store.printStorePath(drvPath),
+            settings.thisSystem,
+            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
+
+    if (drv->isBuiltin())
+        preloadNSS();
+
+#if __APPLE__
+    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
+#endif
+
+    /* Are we doing a chroot build? */
+    {
+        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
+        if (settings.sandboxMode == smEnabled) {
+            if (noChroot)
+                throw Error("derivation '%s' has '__noChroot' set, "
+                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
+#if __APPLE__
+            if (additionalSandboxProfile != "")
+                throw Error("derivation '%s' specifies a sandbox profile, "
+                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
+#endif
+            useChroot = true;
+        }
+        else if (settings.sandboxMode == smDisabled)
+            useChroot = false;
+        else if (settings.sandboxMode == smRelaxed)
+            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
+    }
+
+    if (auto localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
+        auto & localStore = *localStoreP;
+        if (localStore.storeDir != localStore.realStoreDir) {
+            #if __linux__
+                useChroot = true;
+            #else
+                throw Error("building using a diverted store is not supported on this platform");
+            #endif
+        }
+    }
+
+    /* Create a temporary directory where the build will take
+       place. */
+    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+
+    chownToBuilder(tmpDir);
+
+    for (auto & [outputName, status] : initialOutputs) {
+        /* Set scratch path we'll actually use during the build.
+
+           If we're not doing a chroot build, but we have some valid
+           output paths.  Since we can't just overwrite or delete
+           them, we have to do hash rewriting: i.e. in the
+           environment/arguments passed to the build, we replace the
+           hashes of the valid outputs with unique dummy strings;
+           after the build, we discard the redirected outputs
+           corresponding to the valid outputs, and rewrite the
+           contents of the new outputs to replace the dummy strings
+           with the actual hashes. */
+        auto scratchPath =
+            !status.known
+                ? makeFallbackPath(outputName)
+            : !needsHashRewrite()
+                /* Can always use original path in sandbox */
+                ? status.known->path
+            : !status.known->isPresent()
+                /* If path doesn't yet exist can just use it */
+                ? status.known->path
+            : buildMode != bmRepair && !status.known->isValid()
+                /* If we aren't repairing we'll delete a corrupted path, so we
+                   can use original path */
+                ? status.known->path
+            :   /* If we are repairing or the path is totally valid, we'll need
+                   to use a temporary path */
+                makeFallbackPath(status.known->path);
+        scratchOutputs.insert_or_assign(outputName, scratchPath);
+
+        /* A non-removed corrupted path needs to be stored here, too */
+        if (buildMode == bmRepair && !status.known->isValid())
+            redirectedBadOutputs.insert(status.known->path);
+
+        /* Substitute output placeholders with the scratch output paths.
+           We'll use during the build. */
+        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
+
+        /* Additional tasks if we know the final path a priori. */
+        if (!status.known) continue;
+        auto fixedFinalPath = status.known->path;
+
+        /* Additional tasks if the final and scratch are both known and
+           differ. */
+        if (fixedFinalPath == scratchPath) continue;
+
+        /* Ensure scratch path is ours to use. */
+        deletePath(worker.store.printStorePath(scratchPath));
+
+        /* Rewrite and unrewrite paths */
+        {
+            std::string h1 { fixedFinalPath.hashPart() };
+            std::string h2 { scratchPath.hashPart() };
+            inputRewrites[h1] = h2;
+        }
+
+        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
+    }
+
+    /* Construct the environment passed to the builder. */
+    initEnv();
+
+    writeStructuredAttrs();
+
+    /* Handle exportReferencesGraph(), if set. */
+    if (!parsedDrv->getStructuredAttrs()) {
+        /* The `exportReferencesGraph' feature allows the references graph
+           to be passed to a builder.  This attribute should be a list of
+           pairs [name1 path1 name2 path2 ...].  The references graph of
+           each `pathN' will be stored in a text file `nameN' in the
+           temporary build directory.  The text files have the format used
+           by `nix-store --register-validity'.  However, the deriver
+           fields are left empty. */
+        string s = get(drv->env, "exportReferencesGraph").value_or("");
+        Strings ss = tokenizeString<Strings>(s);
+        if (ss.size() % 2 != 0)
+            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
+        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
+            string fileName = *i++;
+            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
+            if (!std::regex_match(fileName, regex))
+                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
+
+            auto storePathS = *i++;
+            if (!worker.store.isInStore(storePathS))
+                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
+            auto storePath = worker.store.toStorePath(storePathS).first;
+
+            /* Write closure info to <fileName>. */
+            writeFile(tmpDir + "/" + fileName,
+                worker.store.makeValidityRegistration(
+                    exportReferences({storePath}), false, false));
+        }
+    }
+
+    if (useChroot) {
+
+        /* Allow a user-configurable set of directories from the
+           host file system. */
+        dirsInChroot.clear();
+
+        for (auto i : settings.sandboxPaths.get()) {
+            if (i.empty()) continue;
+            bool optional = false;
+            if (i[i.size() - 1] == '?') {
+                optional = true;
+                i.pop_back();
+            }
+            size_t p = i.find('=');
+            if (p == string::npos)
+                dirsInChroot[i] = {i, optional};
+            else
+                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
+        }
+        dirsInChroot[tmpDirInSandbox] = tmpDir;
+
+        /* Add the closure of store paths to the chroot. */
+        StorePathSet closure;
+        for (auto & i : dirsInChroot)
+            try {
+                if (worker.store.isInStore(i.second.source))
+                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
+            } catch (InvalidPath & e) {
+            } catch (Error & e) {
+                e.addTrace({}, "while processing 'sandbox-paths'");
+                throw;
+            }
+        for (auto & i : closure) {
+            auto p = worker.store.printStorePath(i);
+            dirsInChroot.insert_or_assign(p, p);
+        }
+
+        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
+
+        /* This works like the above, except on a per-derivation level */
+        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
+
+        for (auto & i : impurePaths) {
+            bool found = false;
+            /* Note: we're not resolving symlinks here to prevent
+               giving a non-root user info about inaccessible
+               files. */
+            Path canonI = canonPath(i);
+            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
+            for (auto & a : allowedPaths) {
+                Path canonA = canonPath(a);
+                if (canonI == canonA || isInDir(canonI, canonA)) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found)
+                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
+                    worker.store.printStorePath(drvPath), i);
+
+            dirsInChroot[i] = i;
+        }
+
+#if __linux__
+        /* Create a temporary directory in which we set up the chroot
+           environment using bind-mounts.  We put it in the Nix store
+           to ensure that we can create hard-links to non-directory
+           inputs in the fake Nix store in the chroot (see below). */
+        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
+        deletePath(chrootRootDir);
+
+        /* Clean up the chroot directory automatically. */
+        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
+
+        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
+
+        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
+            throw SysError("cannot create '%1%'", chrootRootDir);
+
+        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
+            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
+
+        /* Create a writable /tmp in the chroot.  Many builders need
+           this.  (Of course they should really respect $TMPDIR
+           instead.) */
+        Path chrootTmpDir = chrootRootDir + "/tmp";
+        createDirs(chrootTmpDir);
+        chmod_(chrootTmpDir, 01777);
+
+        /* Create a /etc/passwd with entries for the build user and the
+           nobody account.  The latter is kind of a hack to support
+           Samba-in-QEMU. */
+        createDirs(chrootRootDir + "/etc");
+
+        /* Declare the build user's group so that programs get a consistent
+           view of the system (e.g., "id -gn"). */
+        writeFile(chrootRootDir + "/etc/group",
+            fmt("root:x:0:\n"
+                "nixbld:!:%1%:\n"
+                "nogroup:x:65534:\n", sandboxGid()));
+
+        /* Create /etc/hosts with localhost entry. */
+        if (!(derivationIsImpure(derivationType)))
+            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
+
+        /* Make the closure of the inputs available in the chroot,
+           rather than the whole Nix store.  This prevents any access
+           to undeclared dependencies.  Directories are bind-mounted,
+           while other inputs are hard-linked (since only directories
+           can be bind-mounted).  !!! As an extra security
+           precaution, make the fake Nix store only writable by the
+           build user. */
+        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
+        createDirs(chrootStoreDir);
+        chmod_(chrootStoreDir, 01775);
+
+        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
+            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
+
+        for (auto & i : inputPaths) {
+            auto p = worker.store.printStorePath(i);
+            Path r = worker.store.toRealPath(p);
+            if (S_ISDIR(lstat(r).st_mode))
+                dirsInChroot.insert_or_assign(p, r);
+            else
+                linkOrCopy(r, chrootRootDir + p);
+        }
+
+        /* If we're repairing, checking or rebuilding part of a
+           multiple-outputs derivation, it's possible that we're
+           rebuilding a path that is in settings.dirsInChroot
+           (typically the dependencies of /bin/sh).  Throw them
+           out. */
+        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
+            /* If the name isn't known a priori (i.e. floating
+               content-addressed derivation), the temporary location we use
+               should be fresh.  Freshness means it is impossible that the path
+               is already in the sandbox, so we don't need to worry about
+               removing it.  */
+            if (i.second.second)
+                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
+        }
+
+#elif __APPLE__
+        /* We don't really have any parent prep work to do (yet?)
+           All work happens in the child, instead. */
+#else
+        throw Error("sandboxing builds is not supported on this platform");
+#endif
+    }
+
+    if (needsHashRewrite() && pathExists(homeDir))
+        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
+
+    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
+        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
+            % settings.preBuildHook);
+        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
+            Strings({ worker.store.printStorePath(drvPath) });
+        enum BuildHookState {
+            stBegin,
+            stExtraChrootDirs
+        };
+        auto state = stBegin;
+        auto lines = runProgram(settings.preBuildHook, false, args);
+        auto lastPos = std::string::size_type{0};
+        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
+                nlPos = lines.find('\n', lastPos)) {
+            auto line = std::string{lines, lastPos, nlPos - lastPos};
+            lastPos = nlPos + 1;
+            if (state == stBegin) {
+                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
+                    state = stExtraChrootDirs;
+                } else {
+                    throw Error("unknown pre-build hook command '%1%'", line);
+                }
+            } else if (state == stExtraChrootDirs) {
+                if (line == "") {
+                    state = stBegin;
+                } else {
+                    auto p = line.find('=');
+                    if (p == string::npos)
+                        dirsInChroot[line] = line;
+                    else
+                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
+                }
+            }
+        }
+    }
+
+    /* Fire up a Nix daemon to process recursive Nix calls from the
+       builder. */
+    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
+        startDaemon();
+
+    /* Run the builder. */
+    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
+
+    /* Create the log file. */
+    Path logFile = openLogFile();
+
+    /* Create a pipe to get the output of the builder. */
+    //builderOut.create();
+
+    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
+    if (!builderOut.readSide)
+        throw SysError("opening pseudoterminal master");
+
+    std::string slaveName(ptsname(builderOut.readSide.get()));
+
+    if (buildUser) {
+        if (chmod(slaveName.c_str(), 0600))
+            throw SysError("changing mode of pseudoterminal slave");
+
+        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
+            throw SysError("changing owner of pseudoterminal slave");
+    }
+#if __APPLE__
+    else {
+        if (grantpt(builderOut.readSide.get()))
+            throw SysError("granting access to pseudoterminal slave");
+    }
+#endif
+
+    #if 0
+    // Mount the pt in the sandbox so that the "tty" command works.
+    // FIXME: this doesn't work with the new devpts in the sandbox.
+    if (useChroot)
+        dirsInChroot[slaveName] = {slaveName, false};
+    #endif
+
+    if (unlockpt(builderOut.readSide.get()))
+        throw SysError("unlocking pseudoterminal");
+
+    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
+    if (!builderOut.writeSide)
+        throw SysError("opening pseudoterminal slave");
+
+    // Put the pt into raw mode to prevent \n -> \r\n translation.
+    struct termios term;
+    if (tcgetattr(builderOut.writeSide.get(), &term))
+        throw SysError("getting pseudoterminal attributes");
+
+    cfmakeraw(&term);
+
+    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
+        throw SysError("putting pseudoterminal into raw mode");
+
+    result.startTime = time(0);
+
+    /* Fork a child to build the package. */
+    ProcessOptions options;
+
+#if __linux__
+    if (useChroot) {
+        /* Set up private namespaces for the build:
+
+           - The PID namespace causes the build to start as PID 1.
+             Processes outside of the chroot are not visible to those
+             on the inside, but processes inside the chroot are
+             visible from the outside (though with different PIDs).
+
+           - The private mount namespace ensures that all the bind
+             mounts we do will only show up in this process and its
+             children, and will disappear automatically when we're
+             done.
+
+           - The private network namespace ensures that the builder
+             cannot talk to the outside world (or vice versa).  It
+             only has a private loopback interface. (Fixed-output
+             derivations are not run in a private network namespace
+             to allow functions like fetchurl to work.)
+
+           - The IPC namespace prevents the builder from communicating
+             with outside processes using SysV IPC mechanisms (shared
+             memory, message queues, semaphores).  It also ensures
+             that all IPC objects are destroyed when the builder
+             exits.
+
+           - The UTS namespace ensures that builders see a hostname of
+             localhost rather than the actual hostname.
+
+           We use a helper process to do the clone() to work around
+           clone() being broken in multi-threaded programs due to
+           at-fork handlers not being run. Note that we use
+           CLONE_PARENT to ensure that the real builder is parented to
+           us.
+        */
+
+        if (!(derivationIsImpure(derivationType)))
+            privateNetwork = true;
+
+        userNamespaceSync.create();
+
+        options.allowVfork = false;
+
+        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
+        static bool userNamespacesEnabled =
+            pathExists(maxUserNamespaces)
+            && trim(readFile(maxUserNamespaces)) != "0";
+
+        usingUserNamespace = userNamespacesEnabled;
+
+        Pid helper = startProcess([&]() {
+
+            /* Drop additional groups here because we can't do it
+               after we've created the new user namespace.  FIXME:
+               this means that if we're not root in the parent
+               namespace, we can't drop additional groups; they will
+               be mapped to nogroup in the child namespace. There does
+               not seem to be a workaround for this. (But who can tell
+               from reading user_namespaces(7)?)
+               See also https://lwn.net/Articles/621612/. */
+            if (getuid() == 0 && setgroups(0, 0) == -1)
+                throw SysError("setgroups failed");
+
+            size_t stackSize = 1 * 1024 * 1024;
+            char * stack = (char *) mmap(0, stackSize,
+                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+            if (stack == MAP_FAILED) throw SysError("allocating stack");
+
+            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
+            if (privateNetwork)
+                flags |= CLONE_NEWNET;
+            if (usingUserNamespace)
+                flags |= CLONE_NEWUSER;
+
+            pid_t child = clone(childEntry, stack + stackSize, flags, this);
+            if (child == -1 && errno == EINVAL) {
+                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
+                   CLONE_PARENT are not allowed together. */
+                flags &= ~CLONE_NEWPID;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
+                /* Some distros patch Linux to not allow unprivileged
+                 * user namespaces. If we get EPERM or EINVAL, try
+                 * without CLONE_NEWUSER and see if that works.
+                 */
+                usingUserNamespace = false;
+                flags &= ~CLONE_NEWUSER;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            /* Otherwise exit with EPERM so we can handle this in the
+               parent. This is only done when sandbox-fallback is set
+               to true (the default). */
+            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
+                _exit(1);
+            if (child == -1) throw SysError("cloning builder process");
+
+            writeFull(builderOut.writeSide.get(),
+                fmt("%d %d\n", usingUserNamespace, child));
+            _exit(0);
+        }, options);
+
+        int res = helper.wait();
+        if (res != 0 && settings.sandboxFallback) {
+            useChroot = false;
+            initTmpDir();
+            goto fallback;
+        } else if (res != 0)
+            throw Error("unable to start build process");
+
+        userNamespaceSync.readSide = -1;
+
+        /* Close the write side to prevent runChild() from hanging
+           reading from this. */
+        Finally cleanup([&]() {
+            userNamespaceSync.writeSide = -1;
+        });
+
+        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
+        assert(ss.size() == 2);
+        usingUserNamespace = ss[0] == "1";
+        pid = string2Int<pid_t>(ss[1]).value();
+
+        if (usingUserNamespace) {
+            /* Set the UID/GID mapping of the builder's user namespace
+               such that the sandbox user maps to the build user, or to
+               the calling user (if build users are disabled). */
+            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
+            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
+
+            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
+                fmt("%d %d 1", sandboxUid(), hostUid));
+
+            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+
+            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
+                fmt("%d %d 1", sandboxGid(), hostGid));
+        } else {
+            debug("note: not using a user namespace");
+            if (!buildUser)
+                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
+        }
+
+        /* Now that we now the sandbox uid, we can write
+           /etc/passwd. */
+        writeFile(chrootRootDir + "/etc/passwd", fmt(
+                "root:x:0:0:Nix build user:%3%:/noshell\n"
+                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
+                "nobody:x:65534:65534:Nobody:/:/noshell\n",
+                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
+
+        /* Save the mount namespace of the child. We have to do this
+           *before* the child does a chroot. */
+        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
+        if (sandboxMountNamespace.get() == -1)
+            throw SysError("getting sandbox mount namespace");
+
+        /* Signal the builder that we've updated its user namespace. */
+        writeFull(userNamespaceSync.writeSide.get(), "1");
+
+    } else
+#endif
+    {
+    fallback:
+        options.allowVfork = !buildUser && !drv->isBuiltin();
+        pid = startProcess([&]() {
+            runChild();
+        }, options);
+    }
+
+    /* parent */
+    pid.setSeparatePG(true);
+    builderOut.writeSide = -1;
+    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
+
+    /* Check if setting up the build environment failed. */
+    std::vector<std::string> msgs;
+    while (true) {
+        string msg = [&]() {
+            try {
+                return readLine(builderOut.readSide.get());
+            } catch (Error & e) {
+                e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
+                    concatStringsSep("|", msgs));
+                throw e;
+            }
+        }();
+        if (string(msg, 0, 1) == "\2") break;
+        if (string(msg, 0, 1) == "\1") {
+            FdSource source(builderOut.readSide.get());
+            auto ex = readError(source);
+            ex.addTrace({}, "while setting up the build environment");
+            throw ex;
+        }
+        debug("sandbox setup: " + msg);
+        msgs.push_back(std::move(msg));
+    }
+}
+
+
+void DerivationGoal::initTmpDir() {
+    /* In a sandbox, for determinism, always use the same temporary
+       directory. */
+#if __linux__
+    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
+#else
+    tmpDirInSandbox = tmpDir;
+#endif
+
+    /* In non-structured mode, add all bindings specified in the
+       derivation via the environment, except those listed in the
+       passAsFile attribute. Those are passed as file names pointing
+       to temporary files containing the contents. Note that
+       passAsFile is ignored in structure mode because it's not
+       needed (attributes are not passed through the environment, so
+       there is no size constraint). */
+    if (!parsedDrv->getStructuredAttrs()) {
+
+        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
+        for (auto & i : drv->env) {
+            if (passAsFile.find(i.first) == passAsFile.end()) {
+                env[i.first] = i.second;
+            } else {
+                auto hash = hashString(htSHA256, i.first);
+                string fn = ".attr-" + hash.to_string(Base32, false);
+                Path p = tmpDir + "/" + fn;
+                writeFile(p, rewriteStrings(i.second, inputRewrites));
+                chownToBuilder(p);
+                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+            }
+        }
+
+    }
+
+    /* For convenience, set an environment pointing to the top build
+       directory. */
+    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
+
+    /* Also set TMPDIR and variants to point to this directory. */
+    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
+
+    /* Explicitly set PWD to prevent problems with chroot builds.  In
+       particular, dietlibc cannot figure out the cwd because the
+       inode of the current directory doesn't appear in .. (because
+       getdents returns the inode of the mount point). */
+    env["PWD"] = tmpDirInSandbox;
+}
+
+
+void DerivationGoal::initEnv()
+{
+    env.clear();
+
+    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
+       PATH is not set.  We don't want this, so we fill it in with some dummy
+       value. */
+    env["PATH"] = "/path-not-set";
+
+    /* Set HOME to a non-existing path to prevent certain programs from using
+       /etc/passwd (or NIS, or whatever) to locate the home directory (for
+       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
+       if HOME is not set, but they will just assume that the settings file
+       they are looking for does not exist if HOME is set but points to some
+       non-existing path. */
+    env["HOME"] = homeDir;
+
+    /* Tell the builder where the Nix store is.  Usually they
+       shouldn't care, but this is useful for purity checking (e.g.,
+       the compiler or linker might only want to accept paths to files
+       in the store or in the build directory). */
+    env["NIX_STORE"] = worker.store.storeDir;
+
+    /* The maximum number of cores to utilize for parallel building. */
+    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
+
+    initTmpDir();
+
+    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
+       derivation, tell the builder, so that for instance `fetchurl'
+       can skip checking the output.  On older Nixes, this environment
+       variable won't be set, so `fetchurl' will do the check. */
+    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
+
+    /* *Only* if this is a fixed-output derivation, propagate the
+       values of the environment variables specified in the
+       `impureEnvVars' attribute to the builder.  This allows for
+       instance environment variables for proxy configuration such as
+       `http_proxy' to be easily passed to downloaders like
+       `fetchurl'.  Passing such environment variables from the caller
+       to the builder is generally impure, but the output of
+       fixed-output derivations is by definition pure (since we
+       already know the cryptographic hash of the output). */
+    if (derivationIsImpure(derivationType)) {
+        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
+            env[i] = getEnv(i).value_or("");
+    }
+
+    /* Currently structured log messages piggyback on stderr, but we
+       may change that in the future. So tell the builder which file
+       descriptor to use for that. */
+    env["NIX_LOG_FD"] = "2";
+
+    /* Trigger colored output in various tools. */
+    env["TERM"] = "xterm-256color";
+}
+
+
+static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
+
+
+void DerivationGoal::writeStructuredAttrs()
+{
+    auto structuredAttrs = parsedDrv->getStructuredAttrs();
+    if (!structuredAttrs) return;
+
+    auto json = *structuredAttrs;
+
+    /* Add an "outputs" object containing the output paths. */
+    nlohmann::json outputs;
+    for (auto & i : drv->outputs) {
+        /* The placeholder must have a rewrite, so we use it to cover both the
+           cases where we know or don't know the output path ahead of time. */
+        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
+    }
+    json["outputs"] = outputs;
+
+    /* Handle exportReferencesGraph. */
+    auto e = json.find("exportReferencesGraph");
+    if (e != json.end() && e->is_object()) {
+        for (auto i = e->begin(); i != e->end(); ++i) {
+            std::ostringstream str;
+            {
+                JSONPlaceholder jsonRoot(str, true);
+                StorePathSet storePaths;
+                for (auto & p : *i)
+                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
+                worker.store.pathInfoToJSON(jsonRoot,
+                    exportReferences(storePaths), false, true);
+            }
+            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+        }
+    }
+
+    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
+    chownToBuilder(tmpDir + "/.attrs.json");
+
+    /* As a convenience to bash scripts, write a shell file that
+       maps all attributes that are representable in bash -
+       namely, strings, integers, nulls, Booleans, and arrays and
+       objects consisting entirely of those values. (So nested
+       arrays or objects are not supported.) */
+
+    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
+        if (value.is_string())
+            return shellEscape(value);
+
+        if (value.is_number()) {
+            auto f = value.get<float>();
+            if (std::ceil(f) == f)
+                return std::to_string(value.get<int>());
+        }
+
+        if (value.is_null())
+            return std::string("''");
+
+        if (value.is_boolean())
+            return value.get<bool>() ? std::string("1") : std::string("");
+
+        return {};
+    };
+
+    std::string jsonSh;
+
+    for (auto i = json.begin(); i != json.end(); ++i) {
+
+        if (!std::regex_match(i.key(), shVarName)) continue;
+
+        auto & value = i.value();
+
+        auto s = handleSimpleType(value);
+        if (s)
+            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+
+        else if (value.is_array()) {
+            std::string s2;
+            bool good = true;
+
+            for (auto i = value.begin(); i != value.end(); ++i) {
+                auto s3 = handleSimpleType(i.value());
+                if (!s3) { good = false; break; }
+                s2 += *s3; s2 += ' ';
+            }
+
+            if (good)
+                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+        }
+
+        else if (value.is_object()) {
+            std::string s2;
+            bool good = true;
+
+            for (auto i = value.begin(); i != value.end(); ++i) {
+                auto s3 = handleSimpleType(i.value());
+                if (!s3) { good = false; break; }
+                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
+            }
+
+            if (good)
+                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+        }
+    }
+
+    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
+    chownToBuilder(tmpDir + "/.attrs.sh");
+}
+
+struct RestrictedStoreConfig : virtual LocalFSStoreConfig
+{
+    using LocalFSStoreConfig::LocalFSStoreConfig;
+    const std::string name() { return "Restricted Store"; }
+};
+
+/* A wrapper around LocalStore that only allows building/querying of
+   paths that are in the input closures of the build or were added via
+   recursive Nix calls. */
+struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
+{
+    ref<LocalStore> next;
+
+    DerivationGoal & goal;
+
+    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , RestrictedStoreConfig(params)
+        , Store(params)
+        , LocalFSStore(params)
+        , next(next), goal(goal)
+    { }
+
+    Path getRealStoreDir() override
+    { return next->realStoreDir; }
+
+    std::string getUri() override
+    { return next->getUri(); }
+
+    StorePathSet queryAllValidPaths() override
+    {
+        StorePathSet paths;
+        for (auto & p : goal.inputPaths) paths.insert(p);
+        for (auto & p : goal.addedPaths) paths.insert(p);
+        return paths;
+    }
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
+    {
+        if (goal.isAllowed(path)) {
+            try {
+                /* Censor impure information. */
+                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
+                info->deriver.reset();
+                info->registrationTime = 0;
+                info->ultimate = false;
+                info->sigs.clear();
+                callback(info);
+            } catch (InvalidPath &) {
+                callback(nullptr);
+            }
+        } else
+            callback(nullptr);
+    };
+
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
+    { }
+
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
+    {
+        if (!goal.isAllowed(path))
+            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
+        return next->queryPartialDerivationOutputMap(path);
+    }
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+    { throw Error("queryPathFromHashPart"); }
+
+    StorePath addToStore(const string & name, const Path & srcPath,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
+    { throw Error("addToStore"); }
+
+    void addToStore(const ValidPathInfo & info, Source & narSource,
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
+    {
+        next->addToStore(info, narSource, repair, checkSigs);
+        goal.addDependency(info.path);
+    }
+
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair = NoRepair) override
+    {
+        auto path = next->addTextToStore(name, s, references, repair);
+        goal.addDependency(path);
+        return path;
+    }
+
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+    {
+        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
+        goal.addDependency(path);
+        return path;
+    }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    {
+        if (!goal.isAllowed(path))
+            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
+        LocalFSStore::narFromPath(path, sink);
+    }
+
+    void ensurePath(const StorePath & path) override
+    {
+        if (!goal.isAllowed(path))
+            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
+        /* Nothing to be done; 'path' must already be valid. */
+    }
+
+    void registerDrvOutput(const Realisation & info) override
+    // XXX: This should probably be allowed as a no-op if the realisation
+    // corresponds to an allowed derivation
+    { throw Error("registerDrvOutput"); }
+
+    std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
+    // XXX: This should probably be allowed if the realisation corresponds to
+    // an allowed derivation
+    { throw Error("queryRealisation"); }
+
+    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
+    {
+        if (buildMode != bmNormal) throw Error("unsupported build mode");
+
+        StorePathSet newPaths;
+
+        for (auto & path : paths) {
+            if (!goal.isAllowed(path.path))
+                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
+        }
+
+        next->buildPaths(paths, buildMode);
+
+        for (auto & path : paths) {
+            if (!path.path.isDerivation()) continue;
+            auto outputs = next->queryDerivationOutputMap(path.path);
+            for (auto & output : outputs)
+                if (wantOutput(output.first, path.outputs))
+                    newPaths.insert(output.second);
+        }
+
+        StorePathSet closure;
+        next->computeFSClosure(newPaths, closure);
+        for (auto & path : closure)
+            goal.addDependency(path);
+    }
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode = bmNormal) override
+    { unsupported("buildDerivation"); }
+
+    void addTempRoot(const StorePath & path) override
+    { }
+
+    void addIndirectRoot(const Path & path) override
+    { }
+
+    Roots findRoots(bool censor) override
+    { return Roots(); }
+
+    void collectGarbage(const GCOptions & options, GCResults & results) override
+    { }
+
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
+    { unsupported("addSignatures"); }
+
+    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
+        uint64_t & downloadSize, uint64_t & narSize) override
+    {
+        /* This is slightly impure since it leaks information to the
+           client about what paths will be built/substituted or are
+           already present. Probably not a big deal. */
+
+        std::vector<StorePathWithOutputs> allowed;
+        for (auto & path : targets) {
+            if (goal.isAllowed(path.path))
+                allowed.emplace_back(path);
+            else
+                unknown.insert(path.path);
+        }
+
+        next->queryMissing(allowed, willBuild, willSubstitute,
+            unknown, downloadSize, narSize);
+    }
+};
+
+
+void DerivationGoal::startDaemon()
+{
+    settings.requireExperimentalFeature("recursive-nix");
+
+    Store::Params params;
+    params["path-info-cache-size"] = "0";
+    params["store"] = worker.store.storeDir;
+    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
+        params["root"] = localStore->rootDir;
+    params["state"] = "/no-such-path";
+    params["log"] = "/no-such-path";
+    auto store = make_ref<RestrictedStore>(params,
+        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
+        *this);
+
+    addedPaths.clear();
+
+    auto socketName = ".nix-socket";
+    Path socketPath = tmpDir + "/" + socketName;
+    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
+
+    daemonSocket = createUnixDomainSocket(socketPath, 0600);
+
+    chownToBuilder(socketPath);
+
+    daemonThread = std::thread([this, store]() {
+
+        while (true) {
+
+            /* Accept a connection. */
+            struct sockaddr_un remoteAddr;
+            socklen_t remoteAddrLen = sizeof(remoteAddr);
+
+            AutoCloseFD remote = accept(daemonSocket.get(),
+                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
+            if (!remote) {
+                if (errno == EINTR) continue;
+                if (errno == EINVAL) break;
+                throw SysError("accepting connection");
+            }
+
+            closeOnExec(remote.get());
+
+            debug("received daemon connection");
+
+            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
+                FdSource from(remote.get());
+                FdSink to(remote.get());
+                try {
+                    daemon::processConnection(store, from, to,
+                        daemon::NotTrusted, daemon::Recursive,
+                        [&](Store & store) { store.createUser("nobody", 65535); });
+                    debug("terminated daemon connection");
+                } catch (SysError &) {
+                    ignoreException();
+                }
+            });
+
+            daemonWorkerThreads.push_back(std::move(workerThread));
+        }
+
+        debug("daemon shutting down");
+    });
+}
+
+
+void DerivationGoal::stopDaemon()
+{
+    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
+        throw SysError("shutting down daemon socket");
+
+    if (daemonThread.joinable())
+        daemonThread.join();
+
+    // FIXME: should prune worker threads more quickly.
+    // FIXME: shutdown the client socket to speed up worker termination.
+    for (auto & thread : daemonWorkerThreads)
+        thread.join();
+    daemonWorkerThreads.clear();
+
+    daemonSocket = -1;
+}
+
+
+void DerivationGoal::addDependency(const StorePath & path)
+{
+    if (isAllowed(path)) return;
+
+    addedPaths.insert(path);
+
+    /* If we're doing a sandbox build, then we have to make the path
+       appear in the sandbox. */
+    if (useChroot) {
+
+        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
+
+        #if __linux__
+
+            Path source = worker.store.Store::toRealPath(path);
+            Path target = chrootRootDir + worker.store.printStorePath(path);
+            debug("bind-mounting %s -> %s", target, source);
+
+            if (pathExists(target))
+                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
+
+            auto st = lstat(source);
+
+            if (S_ISDIR(st.st_mode)) {
+
+                /* Bind-mount the path into the sandbox. This requires
+                   entering its mount namespace, which is not possible
+                   in multithreaded programs. So we do this in a
+                   child process.*/
+                Pid child(startProcess([&]() {
+
+                    if (setns(sandboxMountNamespace.get(), 0) == -1)
+                        throw SysError("entering sandbox mount namespace");
+
+                    createDirs(target);
+
+                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
+                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
+
+                    _exit(0);
+                }));
+
+                int status = child.wait();
+                if (status != 0)
+                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
+
+            } else
+                linkOrCopy(source, target);
+
+        #else
+            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
+                worker.store.printStorePath(path));
+        #endif
+
+    }
+}
+
+
+void DerivationGoal::chownToBuilder(const Path & path)
+{
+    if (!buildUser) return;
+    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
+        throw SysError("cannot change ownership of '%1%'", path);
+}
+
+
+void setupSeccomp()
+{
+#if __linux__
+    if (!settings.filterSyscalls) return;
+#if HAVE_SECCOMP
+    scmp_filter_ctx ctx;
+
+    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
+        throw SysError("unable to initialize seccomp mode 2");
+
+    Finally cleanup([&]() {
+        seccomp_release(ctx);
+    });
+
+    if (nativeSystem == "x86_64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
+        throw SysError("unable to add 32-bit seccomp architecture");
+
+    if (nativeSystem == "x86_64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
+        throw SysError("unable to add X32 seccomp architecture");
+
+    if (nativeSystem == "aarch64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
+        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
+
+    /* Prevent builders from creating setuid/setgid binaries. */
+    for (int perm : { S_ISUID, S_ISGID }) {
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
+                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+            throw SysError("unable to add seccomp rule");
+
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
+                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+            throw SysError("unable to add seccomp rule");
+
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
+                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+            throw SysError("unable to add seccomp rule");
+    }
+
+    /* Prevent builders from creating EAs or ACLs. Not all filesystems
+       support these, and they're not allowed in the Nix store because
+       they're not representable in the NAR serialisation. */
+    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
+        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
+        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
+        throw SysError("unable to add seccomp rule");
+
+    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
+        throw SysError("unable to set 'no new privileges' seccomp attribute");
+
+    if (seccomp_load(ctx) != 0)
+        throw SysError("unable to load seccomp BPF program");
+#else
+    throw Error(
+        "seccomp is not supported on this platform; "
+        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
+#endif
+#endif
+}
+
+
+void DerivationGoal::runChild()
+{
+    /* Warning: in the child we should absolutely not make any SQLite
+       calls! */
+
+    try { /* child */
+
+        commonChildInit(builderOut);
+
+        try {
+            setupSeccomp();
+        } catch (...) {
+            if (buildUser) throw;
+        }
+
+        bool setUser = true;
+
+        /* Make the contents of netrc available to builtin:fetchurl
+           (which may run under a different uid and/or in a sandbox). */
+        std::string netrcData;
+        try {
+            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
+                netrcData = readFile(settings.netrcFile);
+        } catch (SysError &) { }
+
+#if __linux__
+        if (useChroot) {
+
+            userNamespaceSync.writeSide = -1;
+
+            if (drainFD(userNamespaceSync.readSide.get()) != "1")
+                throw Error("user namespace initialisation failed");
+
+            userNamespaceSync.readSide = -1;
+
+            if (privateNetwork) {
+
+                /* Initialise the loopback interface. */
+                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
+                if (!fd) throw SysError("cannot open IP socket");
+
+                struct ifreq ifr;
+                strcpy(ifr.ifr_name, "lo");
+                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
+                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
+                    throw SysError("cannot set loopback interface flags");
+            }
+
+            /* Set the hostname etc. to fixed values. */
+            char hostname[] = "localhost";
+            if (sethostname(hostname, sizeof(hostname)) == -1)
+                throw SysError("cannot set host name");
+            char domainname[] = "(none)"; // kernel default
+            if (setdomainname(domainname, sizeof(domainname)) == -1)
+                throw SysError("cannot set domain name");
+
+            /* Make all filesystems private.  This is necessary
+               because subtrees may have been mounted as "shared"
+               (MS_SHARED).  (Systemd does this, for instance.)  Even
+               though we have a private mount namespace, mounting
+               filesystems on top of a shared subtree still propagates
+               outside of the namespace.  Making a subtree private is
+               local to the namespace, though, so setting MS_PRIVATE
+               does not affect the outside world. */
+            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
+                throw SysError("unable to make '/' private");
+
+            /* Bind-mount chroot directory to itself, to treat it as a
+               different filesystem from /, as needed for pivot_root. */
+            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
+                throw SysError("unable to bind mount '%1%'", chrootRootDir);
+
+            /* Bind-mount the sandbox's Nix store onto itself so that
+               we can mark it as a "shared" subtree, allowing bind
+               mounts made in *this* mount namespace to be propagated
+               into the child namespace created by the
+               unshare(CLONE_NEWNS) call below.
+
+               Marking chrootRootDir as MS_SHARED causes pivot_root()
+               to fail with EINVAL. Don't know why. */
+            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
+
+            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
+                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
+
+            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
+                throw SysError("unable to make '%s' shared", chrootStoreDir);
+
+            /* Set up a nearly empty /dev, unless the user asked to
+               bind-mount the host /dev. */
+            Strings ss;
+            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
+                createDirs(chrootRootDir + "/dev/shm");
+                createDirs(chrootRootDir + "/dev/pts");
+                ss.push_back("/dev/full");
+                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
+                    ss.push_back("/dev/kvm");
+                ss.push_back("/dev/null");
+                ss.push_back("/dev/random");
+                ss.push_back("/dev/tty");
+                ss.push_back("/dev/urandom");
+                ss.push_back("/dev/zero");
+                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
+                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
+                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
+                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
+            }
+
+            /* Fixed-output derivations typically need to access the
+               network, so give them access to /etc/resolv.conf and so
+               on. */
+            if (derivationIsImpure(derivationType)) {
+                ss.push_back("/etc/resolv.conf");
+
+                // Only use nss functions to resolve hosts and
+                // services. Don’t use it for anything else that may
+                // be configured for this system. This limits the
+                // potential impurities introduced in fixed-outputs.
+                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
+
+                ss.push_back("/etc/services");
+                ss.push_back("/etc/hosts");
+                if (pathExists("/var/run/nscd/socket"))
+                    ss.push_back("/var/run/nscd/socket");
+            }
+
+            for (auto & i : ss) dirsInChroot.emplace(i, i);
+
+            /* Bind-mount all the directories from the "host"
+               filesystem that we want in the chroot
+               environment. */
+            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
+                debug("bind mounting '%1%' to '%2%'", source, target);
+                struct stat st;
+                if (stat(source.c_str(), &st) == -1) {
+                    if (optional && errno == ENOENT)
+                        return;
+                    else
+                        throw SysError("getting attributes of path '%1%'", source);
+                }
+                if (S_ISDIR(st.st_mode))
+                    createDirs(target);
+                else {
+                    createDirs(dirOf(target));
+                    writeFile(target, "");
+                }
+                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
+            };
+
+            for (auto & i : dirsInChroot) {
+                if (i.second.source == "/proc") continue; // backwards compatibility
+                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
+            }
+
+            /* Bind a new instance of procfs on /proc. */
+            createDirs(chrootRootDir + "/proc");
+            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
+                throw SysError("mounting /proc");
+
+            /* Mount a new tmpfs on /dev/shm to ensure that whatever
+               the builder puts in /dev/shm is cleaned up automatically. */
+            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
+                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
+                throw SysError("mounting /dev/shm");
+
+            /* Mount a new devpts on /dev/pts.  Note that this
+               requires the kernel to be compiled with
+               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
+               if /dev/ptx/ptmx exists). */
+            if (pathExists("/dev/pts/ptmx") &&
+                !pathExists(chrootRootDir + "/dev/ptmx")
+                && !dirsInChroot.count("/dev/pts"))
+            {
+                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
+                {
+                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
+
+                    /* Make sure /dev/pts/ptmx is world-writable.  With some
+                       Linux versions, it is created with permissions 0.  */
+                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+                } else {
+                    if (errno != EINVAL)
+                        throw SysError("mounting /dev/pts");
+                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
+                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
+                }
+            }
+
+            /* Unshare this mount namespace. This is necessary because
+               pivot_root() below changes the root of the mount
+               namespace. This means that the call to setns() in
+               addDependency() would hide the host's filesystem,
+               making it impossible to bind-mount paths from the host
+               Nix store into the sandbox. Therefore, we save the
+               pre-pivot_root namespace in
+               sandboxMountNamespace. Since we made /nix/store a
+               shared subtree above, this allows addDependency() to
+               make paths appear in the sandbox. */
+            if (unshare(CLONE_NEWNS) == -1)
+                throw SysError("unsharing mount namespace");
+
+            /* Do the chroot(). */
+            if (chdir(chrootRootDir.c_str()) == -1)
+                throw SysError("cannot change directory to '%1%'", chrootRootDir);
+
+            if (mkdir("real-root", 0) == -1)
+                throw SysError("cannot create real-root directory");
+
+            if (pivot_root(".", "real-root") == -1)
+                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
+
+            if (chroot(".") == -1)
+                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
+
+            if (umount2("real-root", MNT_DETACH) == -1)
+                throw SysError("cannot unmount real root filesystem");
+
+            if (rmdir("real-root") == -1)
+                throw SysError("cannot remove real-root directory");
+
+            /* Switch to the sandbox uid/gid in the user namespace,
+               which corresponds to the build user or calling user in
+               the parent namespace. */
+            if (setgid(sandboxGid()) == -1)
+                throw SysError("setgid failed");
+            if (setuid(sandboxUid()) == -1)
+                throw SysError("setuid failed");
+
+            setUser = false;
+        }
+#endif
+
+        if (chdir(tmpDirInSandbox.c_str()) == -1)
+            throw SysError("changing into '%1%'", tmpDir);
+
+        /* Close all other file descriptors. */
+        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
+
+#if __linux__
+        /* Change the personality to 32-bit if we're doing an
+           i686-linux build on an x86_64-linux machine. */
+        struct utsname utsbuf;
+        uname(&utsbuf);
+        if (drv->platform == "i686-linux" &&
+            (settings.thisSystem == "x86_64-linux" ||
+             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
+            if (personality(PER_LINUX32) == -1)
+                throw SysError("cannot set i686-linux personality");
+        }
+
+        /* Impersonate a Linux 2.6 machine to get some determinism in
+           builds that depend on the kernel version. */
+        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
+            int cur = personality(0xffffffff);
+            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
+        }
+
+        /* Disable address space randomization for improved
+           determinism. */
+        int cur = personality(0xffffffff);
+        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
+#endif
+
+        /* Disable core dumps by default. */
+        struct rlimit limit = { 0, RLIM_INFINITY };
+        setrlimit(RLIMIT_CORE, &limit);
+
+        // FIXME: set other limits to deterministic values?
+
+        /* Fill in the environment. */
+        Strings envStrs;
+        for (auto & i : env)
+            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
+
+        /* If we are running in `build-users' mode, then switch to the
+           user we allocated above.  Make sure that we drop all root
+           privileges.  Note that above we have closed all file
+           descriptors except std*, so that's safe.  Also note that
+           setuid() when run as root sets the real, effective and
+           saved UIDs. */
+        if (setUser && buildUser) {
+            /* Preserve supplementary groups of the build user, to allow
+               admins to specify groups such as "kvm".  */
+            if (!buildUser->getSupplementaryGIDs().empty() &&
+                setgroups(buildUser->getSupplementaryGIDs().size(),
+                          buildUser->getSupplementaryGIDs().data()) == -1)
+                throw SysError("cannot set supplementary groups of build user");
+
+            if (setgid(buildUser->getGID()) == -1 ||
+                getgid() != buildUser->getGID() ||
+                getegid() != buildUser->getGID())
+                throw SysError("setgid failed");
+
+            if (setuid(buildUser->getUID()) == -1 ||
+                getuid() != buildUser->getUID() ||
+                geteuid() != buildUser->getUID())
+                throw SysError("setuid failed");
+        }
+
+        /* Fill in the arguments. */
+        Strings args;
+
+        const char *builder = "invalid";
+
+        if (drv->isBuiltin()) {
+            ;
+        }
+#if __APPLE__
+        else {
+            /* This has to appear before import statements. */
+            std::string sandboxProfile = "(version 1)\n";
+
+            if (useChroot) {
+
+                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
+                PathSet ancestry;
+
+                /* We build the ancestry before adding all inputPaths to the store because we know they'll
+                   all have the same parents (the store), and there might be lots of inputs. This isn't
+                   particularly efficient... I doubt it'll be a bottleneck in practice */
+                for (auto & i : dirsInChroot) {
+                    Path cur = i.first;
+                    while (cur.compare("/") != 0) {
+                        cur = dirOf(cur);
+                        ancestry.insert(cur);
+                    }
+                }
+
+                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
+                   path component this time, since it's typically /nix/store and we care about that. */
+                Path cur = worker.store.storeDir;
+                while (cur.compare("/") != 0) {
+                    ancestry.insert(cur);
+                    cur = dirOf(cur);
+                }
+
+                /* Add all our input paths to the chroot */
+                for (auto & i : inputPaths) {
+                    auto p = worker.store.printStorePath(i);
+                    dirsInChroot[p] = p;
+                }
+
+                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
+                if (settings.darwinLogSandboxViolations) {
+                    sandboxProfile += "(deny default)\n";
+                } else {
+                    sandboxProfile += "(deny default (with no-log))\n";
+                }
+
+                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
+
+                if (derivationIsImpure(derivationType))
+                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
+
+                /* Add the output paths we'll use at build-time to the chroot */
+                sandboxProfile += "(allow file-read* file-write* process-exec\n";
+                for (auto & [_, path] : scratchOutputs)
+                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
+
+                sandboxProfile += ")\n";
+
+                /* Our inputs (transitive dependencies and any impurities computed above)
+
+                   without file-write* allowed, access() incorrectly returns EPERM
+                 */
+                sandboxProfile += "(allow file-read* file-write* process-exec\n";
+                for (auto & i : dirsInChroot) {
+                    if (i.first != i.second.source)
+                        throw Error(
+                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
+                            i.first, i.second.source);
+
+                    string path = i.first;
+                    struct stat st;
+                    if (lstat(path.c_str(), &st)) {
+                        if (i.second.optional && errno == ENOENT)
+                            continue;
+                        throw SysError("getting attributes of path '%s", path);
+                    }
+                    if (S_ISDIR(st.st_mode))
+                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
+                    else
+                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
+                }
+                sandboxProfile += ")\n";
+
+                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
+                sandboxProfile += "(allow file-read*\n";
+                for (auto & i : ancestry) {
+                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
+                }
+                sandboxProfile += ")\n";
+
+                sandboxProfile += additionalSandboxProfile;
+            } else
+                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
+
+            debug("Generated sandbox profile:");
+            debug(sandboxProfile);
+
+            Path sandboxFile = tmpDir + "/.sandbox.sb";
+
+            writeFile(sandboxFile, sandboxProfile);
+
+            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
+
+            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
+               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
+            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
+
+            /* They don't like trailing slashes on subpath directives */
+            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
+
+            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
+                builder = "/usr/bin/sandbox-exec";
+                args.push_back("sandbox-exec");
+                args.push_back("-f");
+                args.push_back(sandboxFile);
+                args.push_back("-D");
+                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
+                args.push_back("-D");
+                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
+                if (allowLocalNetworking) {
+                    args.push_back("-D");
+                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+                }
+                args.push_back(drv->builder);
+            } else {
+                builder = drv->builder.c_str();
+                args.push_back(std::string(baseNameOf(drv->builder)));
+            }
+        }
+#else
+        else {
+            builder = drv->builder.c_str();
+            args.push_back(std::string(baseNameOf(drv->builder)));
+        }
+#endif
+
+        for (auto & i : drv->args)
+            args.push_back(rewriteStrings(i, inputRewrites));
+
+        /* Indicate that we managed to set up the build environment. */
+        writeFull(STDERR_FILENO, string("\2\n"));
+
+        /* Execute the program.  This should not return. */
+        if (drv->isBuiltin()) {
+            try {
+                logger = makeJSONLogger(*logger);
+
+                BasicDerivation & drv2(*drv);
+                for (auto & e : drv2.env)
+                    e.second = rewriteStrings(e.second, inputRewrites);
+
+                if (drv->builder == "builtin:fetchurl")
+                    builtinFetchurl(drv2, netrcData);
+                else if (drv->builder == "builtin:buildenv")
+                    builtinBuildenv(drv2);
+                else if (drv->builder == "builtin:unpack-channel")
+                    builtinUnpackChannel(drv2);
+                else
+                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
+                _exit(0);
+            } catch (std::exception & e) {
+                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
+                _exit(1);
+            }
+        }
+
+#if __APPLE__
+        posix_spawnattr_t attrp;
+
+        if (posix_spawnattr_init(&attrp))
+            throw SysError("failed to initialize builder");
+
+        if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
+            throw SysError("failed to initialize builder");
+
+        if (drv->platform == "aarch64-darwin") {
+            // Unset kern.curproc_arch_affinity so we can escape Rosetta
+            int affinity = 0;
+            sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
+
+            cpu_type_t cpu = CPU_TYPE_ARM64;
+            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+        } else if (drv->platform == "x86_64-darwin") {
+            cpu_type_t cpu = CPU_TYPE_X86_64;
+            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+        }
+
+        posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#else
+        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#endif
+
+        throw SysError("executing '%1%'", drv->builder);
+
+    } catch (Error & e) {
+        writeFull(STDERR_FILENO, "\1\n");
+        FdSink sink(STDERR_FILENO);
+        sink << e;
+        sink.flush();
+        _exit(1);
+    }
+}
+
+
+void DerivationGoal::registerOutputs()
+{
+    /* When using a build hook, the build hook can register the output
+       as valid (by doing `nix-store --import').  If so we don't have
+       to do anything here.
+
+       We can only early return when the outputs are known a priori. For
+       floating content-addressed derivations this isn't the case.
+     */
+    if (hook) {
+        bool allValid = true;
+        for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
+            if (!outputPath || !worker.store.isValidPath(*outputPath))
+                allValid = false;
+            else
+                finalOutputs.insert_or_assign(outputName, *outputPath);
+        }
+        if (allValid) return;
+    }
+
+    std::map<std::string, ValidPathInfo> infos;
+
+    /* Set of inodes seen during calls to canonicalisePathMetaData()
+       for this build's outputs.  This needs to be shared between
+       outputs to allow hard links between outputs. */
+    InodesSeen inodesSeen;
+
+    Path checkSuffix = ".check";
+    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
+
+    std::exception_ptr delayedException;
+
+    /* The paths that can be referenced are the input closures, the
+       output paths, and any paths that have been built via recursive
+       Nix calls. */
+    StorePathSet referenceablePaths;
+    for (auto & p : inputPaths) referenceablePaths.insert(p);
+    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
+    for (auto & p : addedPaths) referenceablePaths.insert(p);
+
+    /* FIXME `needsHashRewrite` should probably be removed and we get to the
+       real reason why we aren't using the chroot dir */
+    auto toRealPathChroot = [&](const Path & p) -> Path {
+        return useChroot && !needsHashRewrite()
+            ? chrootRootDir + p
+            : worker.store.toRealPath(p);
+    };
+
+    /* Check whether the output paths were created, and make all
+       output paths read-only.  Then get the references of each output (that we
+       might need to register), so we can topologically sort them. For the ones
+       that are most definitely already installed, we just store their final
+       name so we can also use it in rewrites. */
+    StringSet outputsToSort;
+    struct AlreadyRegistered { StorePath path; };
+    struct PerhapsNeedToRegister { StorePathSet refs; };
+    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
+    std::map<std::string, struct stat> outputStats;
+    for (auto & [outputName, _] : drv->outputs) {
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
+
+        outputsToSort.insert(outputName);
+
+        /* Updated wanted info to remove the outputs we definitely don't need to register */
+        auto & initialInfo = initialOutputs.at(outputName);
+
+        /* Don't register if already valid, and not checking */
+        initialInfo.wanted = buildMode == bmCheck
+            || !(initialInfo.known && initialInfo.known->isValid());
+        if (!initialInfo.wanted) {
+            outputReferencesIfUnregistered.insert_or_assign(
+                outputName,
+                AlreadyRegistered { .path = initialInfo.known->path });
+            continue;
+        }
+
+        struct stat st;
+        if (lstat(actualPath.c_str(), &st) == -1) {
+            if (errno == ENOENT)
+                throw BuildError(
+                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
+                    worker.store.printStorePath(drvPath), outputName, actualPath);
+            throw SysError("getting attributes of path '%s'", actualPath);
+        }
+
+#ifndef __CYGWIN__
+        /* Check that the output is not group or world writable, as
+           that means that someone else can have interfered with the
+           build.  Also, the output should be owned by the build
+           user. */
+        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
+            (buildUser && st.st_uid != buildUser->getUID()))
+            throw BuildError(
+                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
+                    actualPath, outputName);
+#endif
+
+        /* Canonicalise first.  This ensures that the path we're
+           rewriting doesn't contain a hard link to /etc/shadow or
+           something like that. */
+        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
+
+        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
+
+        /* Pass blank Sink as we are not ready to hash data at this stage. */
+        NullSink blank;
+        auto references = worker.store.parseStorePathSet(
+            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
+
+        outputReferencesIfUnregistered.insert_or_assign(
+            outputName,
+            PerhapsNeedToRegister { .refs = references });
+        outputStats.insert_or_assign(outputName, std::move(st));
+    }
+
+    auto sortedOutputNames = topoSort(outputsToSort,
+        {[&](const std::string & name) {
+            return std::visit(overloaded {
+                /* Since we'll use the already installed versions of these, we
+                   can treat them as leaves and ignore any references they
+                   have. */
+                [&](AlreadyRegistered _) { return StringSet {}; },
+                [&](PerhapsNeedToRegister refs) {
+                    StringSet referencedOutputs;
+                    /* FIXME build inverted map up front so no quadratic waste here */
+                    for (auto & r : refs.refs)
+                        for (auto & [o, p] : scratchOutputs)
+                            if (r == p)
+                                referencedOutputs.insert(o);
+                    return referencedOutputs;
+                },
+            }, outputReferencesIfUnregistered.at(name));
+        }},
+        {[&](const std::string & path, const std::string & parent) {
+            // TODO with more -vvvv also show the temporary paths for manual inspection.
+            return BuildError(
+                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
+                worker.store.printStorePath(drvPath), path, parent);
+        }});
+
+    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
+
+    for (auto & outputName : sortedOutputNames) {
+        auto output = drv->outputs.at(outputName);
+        auto & scratchPath = scratchOutputs.at(outputName);
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
+
+        auto finish = [&](StorePath finalStorePath) {
+            /* Store the final path */
+            finalOutputs.insert_or_assign(outputName, finalStorePath);
+            /* The rewrite rule will be used in downstream outputs that refer to
+               use. This is why the topological sort is essential to do first
+               before this for loop. */
+            if (scratchPath != finalStorePath)
+                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
+        };
+
+        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
+            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
+                finish(skippedFinalPath.path);
+                return std::nullopt;
+            },
+            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
+                return r.refs;
+            },
+        }, outputReferencesIfUnregistered.at(outputName));
+
+        if (!referencesOpt)
+            continue;
+        auto references = *referencesOpt;
+
+        auto rewriteOutput = [&]() {
+            /* Apply hash rewriting if necessary. */
+            if (!outputRewrites.empty()) {
+                warn("rewriting hashes in '%1%'; cross fingers", actualPath);
+
+                /* FIXME: this is in-memory. */
+                StringSink sink;
+                dumpPath(actualPath, sink);
+                deletePath(actualPath);
+                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
+                StringSource source(*sink.s);
+                restorePath(actualPath, source);
+
+                /* FIXME: set proper permissions in restorePath() so
+                   we don't have to do another traversal. */
+                canonicalisePathMetaData(actualPath, -1, inodesSeen);
+            }
+        };
+
+        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
+            /* In the CA case, we need the rewritten refs to calculate the
+               final path, therefore we look for a *non-rewritten
+               self-reference, and use a bool rather try to solve the
+               computationally intractable fixed point. */
+            std::pair<bool, StorePathSet> res {
+                false,
+                {},
+            };
+            for (auto & r : references) {
+                auto name = r.name();
+                auto origHash = std::string { r.hashPart() };
+                if (r == scratchPath)
+                    res.first = true;
+                else if (outputRewrites.count(origHash) == 0)
+                    res.second.insert(r);
+                else {
+                    std::string newRef = outputRewrites.at(origHash);
+                    newRef += '-';
+                    newRef += name;
+                    res.second.insert(StorePath { newRef });
+                }
+            }
+            return res;
+        };
+
+        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
+            auto & st = outputStats.at(outputName);
+            if (outputHash.method == FileIngestionMethod::Flat) {
+                /* The output path should be a regular file without execute permission. */
+                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
+                    throw BuildError(
+                        "output path '%1%' should be a non-executable regular file "
+                        "since recursive hashing is not enabled (outputHashMode=flat)",
+                        actualPath);
+            }
+            rewriteOutput();
+            /* FIXME optimize and deduplicate with addToStore */
+            std::string oldHashPart { scratchPath.hashPart() };
+            HashModuloSink caSink { outputHash.hashType, oldHashPart };
+            switch (outputHash.method) {
+            case FileIngestionMethod::Recursive:
+                dumpPath(actualPath, caSink);
+                break;
+            case FileIngestionMethod::Flat:
+                readFile(actualPath, caSink);
+                break;
+            }
+            auto got = caSink.finish().first;
+            auto refs = rewriteRefs();
+            HashModuloSink narSink { htSHA256, oldHashPart };
+            dumpPath(actualPath, narSink);
+            auto narHashAndSize = narSink.finish();
+            ValidPathInfo newInfo0 {
+                worker.store.makeFixedOutputPath(
+                    outputHash.method,
+                    got,
+                    outputPathName(drv->name, outputName),
+                    refs.second,
+                    refs.first),
+                narHashAndSize.first,
+            };
+            newInfo0.narSize = narHashAndSize.second;
+            newInfo0.ca = FixedOutputHash {
+                .method = outputHash.method,
+                .hash = got,
+            };
+            newInfo0.references = refs.second;
+            if (refs.first)
+                newInfo0.references.insert(newInfo0.path);
+            if (scratchPath != newInfo0.path) {
+                // Also rewrite the output path
+                auto source = sinkToSource([&](Sink & nextSink) {
+                    StringSink sink;
+                    dumpPath(actualPath, sink);
+                    RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
+                    rsink2(*sink.s);
+                    rsink2.flush();
+                });
+                Path tmpPath = actualPath + ".tmp";
+                restorePath(tmpPath, *source);
+                deletePath(actualPath);
+                movePath(tmpPath, actualPath);
+            }
+
+            assert(newInfo0.ca);
+            return newInfo0;
+        };
+
+        ValidPathInfo newInfo = std::visit(overloaded {
+            [&](DerivationOutputInputAddressed output) {
+                /* input-addressed case */
+                auto requiredFinalPath = output.path;
+                /* Preemptively add rewrite rule for final hash, as that is
+                   what the NAR hash will use rather than normalized-self references */
+                if (scratchPath != requiredFinalPath)
+                    outputRewrites.insert_or_assign(
+                        std::string { scratchPath.hashPart() },
+                        std::string { requiredFinalPath.hashPart() });
+                rewriteOutput();
+                auto narHashAndSize = hashPath(htSHA256, actualPath);
+                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
+                newInfo0.narSize = narHashAndSize.second;
+                auto refs = rewriteRefs();
+                newInfo0.references = refs.second;
+                if (refs.first)
+                    newInfo0.references.insert(newInfo0.path);
+                return newInfo0;
+            },
+            [&](DerivationOutputCAFixed dof) {
+                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
+                    .method = dof.hash.method,
+                    .hashType = dof.hash.hash.type,
+                });
+
+                /* Check wanted hash */
+                Hash & wanted = dof.hash.hash;
+                assert(newInfo0.ca);
+                auto got = getContentAddressHash(*newInfo0.ca);
+                if (wanted != got) {
+                    /* Throw an error after registering the path as
+                       valid. */
+                    worker.hashMismatch = true;
+                    delayedException = std::make_exception_ptr(
+                        BuildError("hash mismatch in fixed-output derivation '%s':\n  specified: %s\n     got:    %s",
+                            worker.store.printStorePath(drvPath),
+                            wanted.to_string(SRI, true),
+                            got.to_string(SRI, true)));
+                }
+                return newInfo0;
+            },
+            [&](DerivationOutputCAFloating dof) {
+                return newInfoFromCA(dof);
+            },
+                [&](DerivationOutputDeferred) {
+                // No derivation should reach that point without having been
+                // rewritten first
+                assert(false);
+                // Ugly, but the compiler insists on having this return a value
+                // of type `ValidPathInfo` despite the `assert(false)`, so
+                // let's provide it
+                return *(ValidPathInfo*)0;
+            },
+        }, output.output);
+
+        /* Calculate where we'll move the output files. In the checking case we
+           will leave leave them where they are, for now, rather than move to
+           their usual "final destination" */
+        auto finalDestPath = worker.store.printStorePath(newInfo.path);
+
+        /* Lock final output path, if not already locked. This happens with
+           floating CA derivations and hash-mismatching fixed-output
+           derivations. */
+        PathLocks dynamicOutputLock;
+        auto optFixedPath = output.path(worker.store, drv->name, outputName);
+        if (!optFixedPath ||
+            worker.store.printStorePath(*optFixedPath) != finalDestPath)
+        {
+            assert(newInfo.ca);
+            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
+        }
+
+        /* Move files, if needed */
+        if (worker.store.toRealPath(finalDestPath) != actualPath) {
+            if (buildMode == bmRepair) {
+                /* Path already exists, need to replace it */
+                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
+                actualPath = worker.store.toRealPath(finalDestPath);
+            } else if (buildMode == bmCheck) {
+                /* Path already exists, and we want to compare, so we leave out
+                   new path in place. */
+            } else if (worker.store.isValidPath(newInfo.path)) {
+                /* Path already exists because CA path produced by something
+                   else. No moving needed. */
+                assert(newInfo.ca);
+            } else {
+                auto destPath = worker.store.toRealPath(finalDestPath);
+                movePath(actualPath, destPath);
+                actualPath = destPath;
+            }
+        }
+
+        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
+        if (!localStoreP)
+            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
+        auto & localStore = *localStoreP;
+
+        if (buildMode == bmCheck) {
+
+            if (!worker.store.isValidPath(newInfo.path)) continue;
+            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
+            if (newInfo.narHash != oldInfo.narHash) {
+                worker.checkMismatch = true;
+                if (settings.runDiffHook || settings.keepFailed) {
+                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
+                    deletePath(dst);
+                    movePath(actualPath, dst);
+
+                    handleDiffHook(
+                        buildUser ? buildUser->getUID() : getuid(),
+                        buildUser ? buildUser->getGID() : getgid(),
+                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
+
+                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
+                } else
+                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
+            }
+
+            /* Since we verified the build, it's now ultimately trusted. */
+            if (!oldInfo.ultimate) {
+                oldInfo.ultimate = true;
+                localStore.signPathInfo(oldInfo);
+                localStore.registerValidPaths({{oldInfo.path, oldInfo}});
+            }
+
+            continue;
+        }
+
+        /* For debugging, print out the referenced and unreferenced paths. */
+        for (auto & i : inputPaths) {
+            auto j = references.find(i);
+            if (j == references.end())
+                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
+            else
+                debug("referenced input: '%1%'", worker.store.printStorePath(i));
+        }
+
+        if (curRound == nrRounds) {
+            localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
+            worker.markContentsGood(newInfo.path);
+        }
+
+        newInfo.deriver = drvPath;
+        newInfo.ultimate = true;
+        localStore.signPathInfo(newInfo);
+
+        finish(newInfo.path);
+
+        /* If it's a CA path, register it right away. This is necessary if it
+           isn't statically known so that we can safely unlock the path before
+           the next iteration */
+        if (newInfo.ca)
+            localStore.registerValidPaths({{newInfo.path, newInfo}});
+
+        infos.emplace(outputName, std::move(newInfo));
+    }
+
+    if (buildMode == bmCheck) return;
+
+    /* Apply output checks. */
+    checkOutputs(infos);
+
+    /* Compare the result with the previous round, and report which
+       path is different, if any.*/
+    if (curRound > 1 && prevInfos != infos) {
+        assert(prevInfos.size() == infos.size());
+        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
+            if (!(*i == *j)) {
+                result.isNonDeterministic = true;
+                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
+                bool prevExists = keepPreviousRound && pathExists(prev);
+                hintformat hint = prevExists
+                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
+                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
+                    : hintfmt("output '%s' of '%s' differs from previous round",
+                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
+
+                handleDiffHook(
+                    buildUser ? buildUser->getUID() : getuid(),
+                    buildUser ? buildUser->getGID() : getgid(),
+                    prev, worker.store.printStorePath(i->second.path),
+                    worker.store.printStorePath(drvPath), tmpDir);
+
+                if (settings.enforceDeterminism)
+                    throw NotDeterministic(hint);
+
+                printError(hint);
+
+                curRound = nrRounds; // we know enough, bail out early
+            }
+    }
+
+    /* If this is the first round of several, then move the output out of the way. */
+    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
+        for (auto & [_, outputStorePath] : finalOutputs) {
+            auto path = worker.store.printStorePath(outputStorePath);
+            Path prev = path + checkSuffix;
+            deletePath(prev);
+            Path dst = path + checkSuffix;
+            if (rename(path.c_str(), dst.c_str()))
+                throw SysError("renaming '%s' to '%s'", path, dst);
+        }
+    }
+
+    if (curRound < nrRounds) {
+        prevInfos = std::move(infos);
+        return;
+    }
+
+    /* Remove the .check directories if we're done. FIXME: keep them
+       if the result was not determistic? */
+    if (curRound == nrRounds) {
+        for (auto & [_, outputStorePath] : finalOutputs) {
+            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
+            deletePath(prev);
+        }
+    }
+
+    /* Register each output path as valid, and register the sets of
+       paths referenced by each of them.  If there are cycles in the
+       outputs, this will fail. */
+    {
+        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
+        if (!localStoreP)
+            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
+        auto & localStore = *localStoreP;
+
+        ValidPathInfos infos2;
+        for (auto & [outputName, newInfo] : infos) {
+            infos2.insert_or_assign(newInfo.path, newInfo);
+        }
+        localStore.registerValidPaths(infos2);
+    }
+
+    /* In case of a fixed-output derivation hash mismatch, throw an
+       exception now that we have registered the output as valid. */
+    if (delayedException)
+        std::rethrow_exception(delayedException);
+
+    /* If we made it this far, we are sure the output matches the derivation
+       (since the delayedException would be a fixed output CA mismatch). That
+       means it's safe to link the derivation to the output hash. We must do
+       that for floating CA derivations, which otherwise couldn't be cached,
+       but it's fine to do in all cases. */
+
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        for (auto& [outputName, newInfo] : infos)
+            worker.store.registerDrvOutput(Realisation{
+                .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName},
+                .outPath = newInfo.path});
+    }
+}
+
+
+void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
+{
+    std::map<Path, const ValidPathInfo &> outputsByPath;
+    for (auto & output : outputs)
+        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
+
+    for (auto & output : outputs) {
+        auto & outputName = output.first;
+        auto & info = output.second;
+
+        struct Checks
+        {
+            bool ignoreSelfRefs = false;
+            std::optional<uint64_t> maxSize, maxClosureSize;
+            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
+        };
+
+        /* Compute the closure and closure size of some output. This
+           is slightly tricky because some of its references (namely
+           other outputs) may not be valid yet. */
+        auto getClosure = [&](const StorePath & path)
+        {
+            uint64_t closureSize = 0;
+            StorePathSet pathsDone;
+            std::queue<StorePath> pathsLeft;
+            pathsLeft.push(path);
+
+            while (!pathsLeft.empty()) {
+                auto path = pathsLeft.front();
+                pathsLeft.pop();
+                if (!pathsDone.insert(path).second) continue;
+
+                auto i = outputsByPath.find(worker.store.printStorePath(path));
+                if (i != outputsByPath.end()) {
+                    closureSize += i->second.narSize;
+                    for (auto & ref : i->second.references)
+                        pathsLeft.push(ref);
+                } else {
+                    auto info = worker.store.queryPathInfo(path);
+                    closureSize += info->narSize;
+                    for (auto & ref : info->references)
+                        pathsLeft.push(ref);
+                }
+            }
+
+            return std::make_pair(std::move(pathsDone), closureSize);
+        };
+
+        auto applyChecks = [&](const Checks & checks)
+        {
+            if (checks.maxSize && info.narSize > *checks.maxSize)
+                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
+                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
+
+            if (checks.maxClosureSize) {
+                uint64_t closureSize = getClosure(info.path).second;
+                if (closureSize > *checks.maxClosureSize)
+                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
+                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
+            }
+
+            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
+            {
+                if (!value) return;
+
+                /* Parse a list of reference specifiers.  Each element must
+                   either be a store path, or the symbolic name of the output
+                   of the derivation (such as `out'). */
+                StorePathSet spec;
+                for (auto & i : *value) {
+                    if (worker.store.isStorePath(i))
+                        spec.insert(worker.store.parseStorePath(i));
+                    else if (finalOutputs.count(i))
+                        spec.insert(finalOutputs.at(i));
+                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
+                }
+
+                auto used = recursive
+                    ? getClosure(info.path).first
+                    : info.references;
+
+                if (recursive && checks.ignoreSelfRefs)
+                    used.erase(info.path);
+
+                StorePathSet badPaths;
+
+                for (auto & i : used)
+                    if (allowed) {
+                        if (!spec.count(i))
+                            badPaths.insert(i);
+                    } else {
+                        if (spec.count(i))
+                            badPaths.insert(i);
+                    }
+
+                if (!badPaths.empty()) {
+                    string badPathsStr;
+                    for (auto & i : badPaths) {
+                        badPathsStr += "\n  ";
+                        badPathsStr += worker.store.printStorePath(i);
+                    }
+                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
+                        worker.store.printStorePath(info.path), badPathsStr);
+                }
+            };
+
+            checkRefs(checks.allowedReferences, true, false);
+            checkRefs(checks.allowedRequisites, true, true);
+            checkRefs(checks.disallowedReferences, false, false);
+            checkRefs(checks.disallowedRequisites, false, true);
+        };
+
+        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
+            auto outputChecks = structuredAttrs->find("outputChecks");
+            if (outputChecks != structuredAttrs->end()) {
+                auto output = outputChecks->find(outputName);
+
+                if (output != outputChecks->end()) {
+                    Checks checks;
+
+                    auto maxSize = output->find("maxSize");
+                    if (maxSize != output->end())
+                        checks.maxSize = maxSize->get<uint64_t>();
+
+                    auto maxClosureSize = output->find("maxClosureSize");
+                    if (maxClosureSize != output->end())
+                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
+
+                    auto get = [&](const std::string & name) -> std::optional<Strings> {
+                        auto i = output->find(name);
+                        if (i != output->end()) {
+                            Strings res;
+                            for (auto j = i->begin(); j != i->end(); ++j) {
+                                if (!j->is_string())
+                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
+                                res.push_back(j->get<std::string>());
+                            }
+                            checks.disallowedRequisites = res;
+                            return res;
+                        }
+                        return {};
+                    };
+
+                    checks.allowedReferences = get("allowedReferences");
+                    checks.allowedRequisites = get("allowedRequisites");
+                    checks.disallowedReferences = get("disallowedReferences");
+                    checks.disallowedRequisites = get("disallowedRequisites");
+
+                    applyChecks(checks);
+                }
+            }
+        } else {
+            // legacy non-structured-attributes case
+            Checks checks;
+            checks.ignoreSelfRefs = true;
+            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
+            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
+            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
+            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
+            applyChecks(checks);
+        }
+    }
+}
+
+
+Path DerivationGoal::openLogFile()
+{
+    logSize = 0;
+
+    if (!settings.keepLog) return "";
+
+    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
+
+    /* Create a log file. */
+    Path logDir;
+    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
+        logDir = localStore->logDir;
+    else
+        logDir = settings.nixLogDir;
+    Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
+    createDirs(dir);
+
+    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
+        settings.compressLog ? ".bz2" : "");
+
+    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
+    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
+
+    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
+
+    if (settings.compressLog)
+        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
+    else
+        logSink = logFileSink;
+
+    return logFileName;
+}
+
+
+void DerivationGoal::closeLogFile()
+{
+    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
+    if (logSink2) logSink2->finish();
+    if (logFileSink) logFileSink->flush();
+    logSink = logFileSink = 0;
+    fdLogFile = -1;
+}
+
+
+void DerivationGoal::deleteTmpDir(bool force)
+{
+    if (tmpDir != "") {
+        /* Don't keep temporary directories for builtins because they
+           might have privileged stuff (like a copy of netrc). */
+        if (settings.keepFailed && !force && !drv->isBuiltin()) {
+            printError("note: keeping build directory '%s'", tmpDir);
+            chmod(tmpDir.c_str(), 0755);
+        }
+        else
+            deletePath(tmpDir);
+        tmpDir = "";
+    }
+}
+
+
+void DerivationGoal::handleChildOutput(int fd, const string & data)
+{
+    if ((hook && fd == hook->builderOut.readSide.get()) ||
+        (!hook && fd == builderOut.readSide.get()))
+    {
+        logSize += data.size();
+        if (settings.maxLogSize && logSize > settings.maxLogSize) {
+            killChild();
+            done(
+                BuildResult::LogLimitExceeded,
+                Error("%s killed after writing more than %d bytes of log output",
+                    getName(), settings.maxLogSize));
+            return;
+        }
+
+        for (auto c : data)
+            if (c == '\r')
+                currentLogLinePos = 0;
+            else if (c == '\n')
+                flushLine();
+            else {
+                if (currentLogLinePos >= currentLogLine.size())
+                    currentLogLine.resize(currentLogLinePos + 1);
+                currentLogLine[currentLogLinePos++] = c;
+            }
+
+        if (logSink) (*logSink)(data);
+    }
+
+    if (hook && fd == hook->fromHook.readSide.get()) {
+        for (auto c : data)
+            if (c == '\n') {
+                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
+                currentHookLine.clear();
+            } else
+                currentHookLine += c;
+    }
+}
+
+
+void DerivationGoal::handleEOF(int fd)
+{
+    if (!currentLogLine.empty()) flushLine();
+    worker.wakeUp(shared_from_this());
+}
+
+
+void DerivationGoal::flushLine()
+{
+    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
+        ;
+
+    else {
+        logTail.push_back(currentLogLine);
+        if (logTail.size() > settings.logLines) logTail.pop_front();
+
+        act->result(resBuildLogLine, currentLogLine);
+    }
+
+    currentLogLine = "";
+    currentLogLinePos = 0;
+}
+
+
+std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
+{
+    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+        std::map<std::string, std::optional<StorePath>> res;
+        for (auto & [name, output] : drv->outputs)
+            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
+        return res;
+    } else {
+        return worker.store.queryPartialDerivationOutputMap(drvPath);
+    }
+}
+
+OutputPathMap DerivationGoal::queryDerivationOutputMap()
+{
+    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+        OutputPathMap res;
+        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
+            res.insert_or_assign(name, *output.second);
+        return res;
+    } else {
+        return worker.store.queryDerivationOutputMap(drvPath);
+    }
+}
+
+
+void DerivationGoal::checkPathValidity()
+{
+    bool checkHash = buildMode == bmRepair;
+    for (auto & i : queryPartialDerivationOutputMap()) {
+        InitialOutput & info = initialOutputs.at(i.first);
+        info.wanted = wantOutput(i.first, wantedOutputs);
+        if (i.second) {
+            auto outputPath = *i.second;
+            info.known = {
+                .path = outputPath,
+                .status = !worker.store.isValidPath(outputPath)
+                    ? PathStatus::Absent
+                    : !checkHash || worker.pathContentsGood(outputPath)
+                    ? PathStatus::Valid
+                    : PathStatus::Corrupt,
+            };
+        }
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            if (auto real = worker.store.queryRealisation(
+                    DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) {
+                info.known = {
+                    .path = real->outPath,
+                    .status = PathStatus::Valid,
+                };
+            }
+        }
+    }
+}
+
+
+StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
+{
+    return worker.store.makeStorePath(
+        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
+        Hash(htSHA256), outputPathName(drv->name, outputName));
+}
+
+
+StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
+{
+    return worker.store.makeStorePath(
+        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
+        Hash(htSHA256), path.name());
+}
+
+
+void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
+{
+    result.status = status;
+    if (ex)
+        result.errorMsg = ex->what();
+    amDone(result.success() ? ecSuccess : ecFailed, ex);
+    if (result.status == BuildResult::TimedOut)
+        worker.timedOut = true;
+    if (result.status == BuildResult::PermanentFailure)
+        worker.permanentFailure = true;
+
+    mcExpectedBuilds.reset();
+    mcRunningBuilds.reset();
+
+    if (result.success()) {
+        if (status == BuildResult::Built)
+            worker.doneBuilds++;
+    } else {
+        if (status != BuildResult::DependencyFailed)
+            worker.failedBuilds++;
+    }
+
+    worker.updateProgress();
+}
+
+
+}
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
new file mode 100644
index 000000000..6dc164922
--- /dev/null
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -0,0 +1,373 @@
+#pragma once
+
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "local-store.hh"
+#include "goal.hh"
+
+namespace nix {
+
+using std::map;
+
+struct HookInstance;
+
+typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
+
+/* Unless we are repairing, we don't both to test validity and just assume it,
+   so the choices are `Absent` or `Valid`. */
+enum struct PathStatus {
+    Corrupt,
+    Absent,
+    Valid,
+};
+
+struct InitialOutputStatus {
+    StorePath path;
+    PathStatus status;
+    /* Valid in the store, and additionally non-corrupt if we are repairing */
+    bool isValid() const {
+        return status == PathStatus::Valid;
+    }
+    /* Merely present, allowed to be corrupt */
+    bool isPresent() const {
+        return status == PathStatus::Corrupt
+            || status == PathStatus::Valid;
+    }
+};
+
+struct InitialOutput {
+    bool wanted;
+    Hash outputHash;
+    std::optional<InitialOutputStatus> known;
+};
+
+struct DerivationGoal : public Goal
+{
+    /* Whether to use an on-disk .drv file. */
+    bool useDerivation;
+
+    /* The path of the derivation. */
+    StorePath drvPath;
+
+    /* The path of the corresponding resolved derivation */
+    std::optional<BasicDerivation> resolvedDrv;
+
+    /* The specific outputs that we need to build.  Empty means all of
+       them. */
+    StringSet wantedOutputs;
+
+    /* Whether additional wanted outputs have been added. */
+    bool needRestart = false;
+
+    /* Whether to retry substituting the outputs after building the
+       inputs. */
+    bool retrySubstitution;
+
+    /* The derivation stored at drvPath. */
+    std::unique_ptr<Derivation> drv;
+
+    std::unique_ptr<ParsedDerivation> parsedDrv;
+
+    /* The remainder is state held during the build. */
+
+    /* Locks on (fixed) output paths. */
+    PathLocks outputLocks;
+
+    /* All input paths (that is, the union of FS closures of the
+       immediate input paths). */
+    StorePathSet inputPaths;
+
+    std::map<std::string, InitialOutput> initialOutputs;
+
+    /* User selected for running the builder. */
+    std::unique_ptr<UserLock> buildUser;
+
+    /* The process ID of the builder. */
+    Pid pid;
+
+    /* The temporary directory. */
+    Path tmpDir;
+
+    /* The path of the temporary directory in the sandbox. */
+    Path tmpDirInSandbox;
+
+    /* File descriptor for the log file. */
+    AutoCloseFD fdLogFile;
+    std::shared_ptr<BufferedSink> logFileSink, logSink;
+
+    /* Number of bytes received from the builder's stdout/stderr. */
+    unsigned long logSize;
+
+    /* The most recent log lines. */
+    std::list<std::string> logTail;
+
+    std::string currentLogLine;
+    size_t currentLogLinePos = 0; // to handle carriage return
+
+    std::string currentHookLine;
+
+    /* Pipe for the builder's standard output/error. */
+    Pipe builderOut;
+
+    /* Pipe for synchronising updates to the builder namespaces. */
+    Pipe userNamespaceSync;
+
+    /* The mount namespace of the builder, used to add additional
+       paths to the sandbox as a result of recursive Nix calls. */
+    AutoCloseFD sandboxMountNamespace;
+
+    /* On Linux, whether we're doing the build in its own user
+       namespace. */
+    bool usingUserNamespace = true;
+
+    /* The build hook. */
+    std::unique_ptr<HookInstance> hook;
+
+    /* Whether we're currently doing a chroot build. */
+    bool useChroot = false;
+
+    Path chrootRootDir;
+
+    /* RAII object to delete the chroot directory. */
+    std::shared_ptr<AutoDelete> autoDelChroot;
+
+    /* The sort of derivation we are building. */
+    DerivationType derivationType;
+
+    /* Whether to run the build in a private network namespace. */
+    bool privateNetwork = false;
+
+    typedef void (DerivationGoal::*GoalState)();
+    GoalState state;
+
+    /* Stuff we need to pass to initChild(). */
+    struct ChrootPath {
+        Path source;
+        bool optional;
+        ChrootPath(Path source = "", bool optional = false)
+            : source(source), optional(optional)
+        { }
+    };
+    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
+    DirsInChroot dirsInChroot;
+
+    typedef map<string, string> Environment;
+    Environment env;
+
+#if __APPLE__
+    typedef string SandboxProfile;
+    SandboxProfile additionalSandboxProfile;
+#endif
+
+    /* Hash rewriting. */
+    StringMap inputRewrites, outputRewrites;
+    typedef map<StorePath, StorePath> RedirectedOutputs;
+    RedirectedOutputs redirectedOutputs;
+
+    /* The outputs paths used during the build.
+
+       - Input-addressed derivations or fixed content-addressed outputs are
+         sometimes built when some of their outputs already exist, and can not
+         be hidden via sandboxing. We use temporary locations instead and
+         rewrite after the build. Otherwise the regular predetermined paths are
+         put here.
+
+       - Floating content-addressed derivations do not know their final build
+         output paths until the outputs are hashed, so random locations are
+         used, and then renamed. The randomness helps guard against hidden
+         self-references.
+     */
+    OutputPathMap scratchOutputs;
+
+    /* The final output paths of the build.
+
+       - For input-addressed derivations, always the precomputed paths
+
+       - For content-addressed derivations, calcuated from whatever the hash
+         ends up being. (Note that fixed outputs derivations that produce the
+         "wrong" output still install that data under its true content-address.)
+     */
+    OutputPathMap finalOutputs;
+
+    BuildMode buildMode;
+
+    /* If we're repairing without a chroot, there may be outputs that
+       are valid but corrupt.  So we redirect these outputs to
+       temporary paths. */
+    StorePathSet redirectedBadOutputs;
+
+    BuildResult result;
+
+    /* The current round, if we're building multiple times. */
+    size_t curRound = 1;
+
+    size_t nrRounds;
+
+    /* Path registration info from the previous round, if we're
+       building multiple times. Since this contains the hash, it
+       allows us to compare whether two rounds produced the same
+       result. */
+    std::map<Path, ValidPathInfo> prevInfos;
+
+    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
+
+    const static Path homeDir;
+
+    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
+
+    std::unique_ptr<Activity> act;
+
+    /* Activity that denotes waiting for a lock. */
+    std::unique_ptr<Activity> actLock;
+
+    std::map<ActivityId, Activity> builderActivities;
+
+    /* The remote machine on which we're building. */
+    std::string machineName;
+
+    /* The recursive Nix daemon socket. */
+    AutoCloseFD daemonSocket;
+
+    /* The daemon main thread. */
+    std::thread daemonThread;
+
+    /* The daemon worker threads. */
+    std::vector<std::thread> daemonWorkerThreads;
+
+    /* Paths that were added via recursive Nix calls. */
+    StorePathSet addedPaths;
+
+    /* Recursive Nix calls are only allowed to build or realize paths
+       in the original input closure or added via a recursive Nix call
+       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+       /nix/store/<bla> is some arbitrary path in a binary cache). */
+    bool isAllowed(const StorePath & path)
+    {
+        return inputPaths.count(path) || addedPaths.count(path);
+    }
+
+    friend struct RestrictedStore;
+
+    DerivationGoal(const StorePath & drvPath,
+        const StringSet & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
+    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
+        const StringSet & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
+    ~DerivationGoal();
+
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite();
+
+    void timedOut(Error && ex) override;
+
+    string key() override;
+
+    void work() override;
+
+    /* Add wanted outputs to an already existing derivation goal. */
+    void addWantedOutputs(const StringSet & outputs);
+
+    BuildResult getResult() { return result; }
+
+    /* The states. */
+    void getDerivation();
+    void loadDerivation();
+    void haveDerivation();
+    void outputsSubstitutionTried();
+    void gaveUpOnSubstitution();
+    void closureRepaired();
+    void inputsRealised();
+    void tryToBuild();
+    void tryLocalBuild();
+    void buildDone();
+
+    void resolvedFinished();
+
+    /* Is the build hook willing to perform the build? */
+    HookReply tryBuildHook();
+
+    /* Start building a derivation. */
+    void startBuilder();
+
+    /* Fill in the environment for the builder. */
+    void initEnv();
+
+    /* Setup tmp dir location. */
+    void initTmpDir();
+
+    /* Write a JSON file containing the derivation attributes. */
+    void writeStructuredAttrs();
+
+    void startDaemon();
+
+    void stopDaemon();
+
+    /* Add 'path' to the set of paths that may be referenced by the
+       outputs, and make it appear in the sandbox. */
+    void addDependency(const StorePath & path);
+
+    /* Make a file owned by the builder. */
+    void chownToBuilder(const Path & path);
+
+    /* Run the builder's process. */
+    void runChild();
+
+    /* Check that the derivation outputs all exist and register them
+       as valid. */
+    void registerOutputs();
+
+    /* Check that an output meets the requirements specified by the
+       'outputChecks' attribute (or the legacy
+       '{allowed,disallowed}{References,Requisites}' attributes). */
+    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
+    /* Open a log file and a pipe to it. */
+    Path openLogFile();
+
+    /* Close the log file. */
+    void closeLogFile();
+
+    /* Delete the temporary directory, if we have one. */
+    void deleteTmpDir(bool force);
+
+    /* Callback used by the worker to write to the log. */
+    void handleChildOutput(int fd, const string & data) override;
+    void handleEOF(int fd) override;
+    void flushLine();
+
+    /* Wrappers around the corresponding Store methods that first consult the
+       derivation.  This is currently needed because when there is no drv file
+       there also is no DB entry. */
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
+    OutputPathMap queryDerivationOutputMap();
+
+    /* Return the set of (in)valid paths. */
+    void checkPathValidity();
+
+    /* Forcibly kill the child process, if any. */
+    void killChild();
+
+    /* Create alternative path calculated from but distinct from the
+       input, so we can avoid overwriting outputs (or other store paths)
+       that already exist. */
+    StorePath makeFallbackPath(const StorePath & path);
+    /* Make a path to another based on the output name along with the
+       derivation hash. */
+    /* FIXME add option to randomize, so we can audit whether our
+       rewrites caught everything */
+    StorePath makeFallbackPath(std::string_view outputName);
+
+    void repairClosure();
+
+    void started();
+
+    void done(
+        BuildResult::Status status,
+        std::optional<Error> ex = {});
+
+    StorePathSet exportReferences(const StorePathSet & storePaths);
+};
+
+}

From 68f4c728eca33f115f90e3f924c9081a4cd59896 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 Feb 2021 15:20:33 +0000
Subject: [PATCH 776/882] Split {,local-}derivation-goal.{cc,hh}

This separates the scheduling logic (including simple hook pathway) from
the local-store needing code.

This should be the final split for now. I'm reasonably happy with how
it's turning out, even before I'm done moving code into
`local-derivation-goal`. Benefits:

1. This will help "witness" that the hook case is indeed a lot simpler,
   and also compensate for the increased complexity that comes from
   content-addressed derivation outputs.

2. It also moves us ever so slightly towards a world where we could use
   off-the-shelf storage or sandboxing, since `local-derivation-goal`
   would be gutted in those cases, but `derivation-goal` should remain
   nearly the same.

The new `#if 0` in the new files will be deleted in the following
commit. I keep it here so if it turns out more stuff can be moved over,
it's easy to do so in a way that preserves ordering --- and thus
prevents conflicts.

N.B.
```sh
git diff HEAD^^ --color-moved --find-copies-harder --patience --stat
```
makes nicer output.
---
 src/libstore/build/derivation-goal.cc       | 2741 +------------------
 src/libstore/build/derivation-goal.hh       |  186 +-
 src/libstore/build/entry-points.cc          |    1 +
 src/libstore/build/local-derivation-goal.cc |  330 +--
 src/libstore/build/local-derivation-goal.hh |  100 +-
 src/libstore/build/worker.cc                |   14 +-
 src/libstore/local-store.hh                 |    2 +-
 7 files changed, 329 insertions(+), 3045 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 924c69fb7..c29237f5c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -9,10 +9,10 @@
 #include "archive.hh"
 #include "json.hh"
 #include "compression.hh"
-#include "daemon.hh"
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
+#include "local-store.hh" // TODO remove, along with remaining downcasts
 
 #include <regex>
 #include <queue>
@@ -62,40 +62,6 @@
 
 namespace nix {
 
-void handleDiffHook(
-    uid_t uid, uid_t gid,
-    const Path & tryA, const Path & tryB,
-    const Path & drvPath, const Path & tmpDir)
-{
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
-        try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-
-            auto diffRes = runProgram(diffHookOptions);
-            if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first,
-                    "diff-hook program '%1%' %2%",
-                    diffHook,
-                    statusToString(diffRes.first));
-
-            if (diffRes.second != "")
-                printError(chomp(diffRes.second));
-        } catch (Error & error) {
-            ErrorInfo ei = error.info();
-            // FIXME: wrap errors.
-            ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str());
-            logError(ei);
-        }
-    }
-}
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker)
@@ -144,9 +110,6 @@ DerivationGoal::~DerivationGoal()
 {
     /* Careful: we should never ever throw an exception from a
        destructor. */
-    try { killChild(); } catch (...) { ignoreException(); }
-    try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
     try { closeLogFile(); } catch (...) { ignoreException(); }
 }
 
@@ -161,38 +124,8 @@ string DerivationGoal::key()
 }
 
 
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
-    return !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    return true;
-#endif
-}
-
-
 void DerivationGoal::killChild()
 {
-    if (pid != -1) {
-        worker.childTerminated(this);
-
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
-
-        assert(pid == -1);
-    }
-
     hook.reset();
 }
 
@@ -697,64 +630,10 @@ void DerivationGoal::tryToBuild()
 }
 
 void DerivationGoal::tryLocalBuild() {
-    /* Make sure that we are allowed to start a build. */
-    if (!dynamic_cast<LocalStore *>(&worker.store)) {
-        throw Error(
-            "unable to build with a primary store that isn't a local store; "
-            "either pass a different '--store' or enable remote builds."
-            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-    }
-    unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs) {
-        worker.waitForBuildSlot(shared_from_this());
-        outputLocks.unlock();
-        return;
-    }
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
-            if (!actLock)
-                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-            worker.waitForAWhile(shared_from_this());
-            return;
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
-    actLock.reset();
-
-    try {
-
-        /* Okay, we have to build. */
-        startBuilder();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-        buildUser.reset();
-        worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
-        return;
-    }
-
-    /* This state will be reached when we get EOF on the child's
-       log pipe. */
-    state = &DerivationGoal::buildDone;
-
-    started();
+    throw Error(
+        "unable to build with a primary store that isn't a local store; "
+        "either pass a different '--store' or enable remote builds."
+        "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
 }
 
 
@@ -811,25 +690,63 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
 }
 
 
-MakeError(NotDeterministic, BuildError);
+int DerivationGoal::getChildStatus()
+{
+    return hook->pid.kill();
+}
+
+
+void DerivationGoal::closeReadPipes()
+{
+    hook->builderOut.readSide = -1;
+    hook->fromHook.readSide = -1;
+}
+
+
+void DerivationGoal::cleanupHookFinally()
+{
+}
+
+
+void DerivationGoal::cleanupPreChildKill()
+{
+}
+
+
+void DerivationGoal::cleanupPostChildKill()
+{
+}
+
+
+bool DerivationGoal::cleanupDecideWhetherDiskFull()
+{
+    return false;
+}
+
+
+void DerivationGoal::cleanupPostOutputsRegisteredModeCheck()
+{
+}
+
+
+void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
+{
+}
 
 
 void DerivationGoal::buildDone()
 {
     trace("build done");
 
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
+    Finally releaseBuildUser([&](){ this->cleanupHookFinally(); });
 
-    sandboxMountNamespace = -1;
+    cleanupPreChildKill();
 
     /* Since we got an EOF on the logger pipe, the builder is presumed
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe, so just to be sure,
        kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
+    int status = getChildStatus();
 
     debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
 
@@ -840,24 +757,12 @@ void DerivationGoal::buildDone()
     worker.childTerminated(this);
 
     /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
+    closeReadPipes();
 
     /* Close the log file. */
     closeLogFile();
 
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
+    cleanupPostChildKill();
 
     bool diskFull = false;
 
@@ -866,36 +771,7 @@ void DerivationGoal::buildDone()
         /* Check the exit status. */
         if (!statusOk(status)) {
 
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            if (auto localStore = dynamic_cast<LocalStore *>(&worker.store)) {
-                uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-                struct statvfs st;
-                if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 &&
-                    (uint64_t) st.f_bavail * st.f_bsize < required)
-                    diskFull = true;
-                if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                    (uint64_t) st.f_bavail * st.f_bsize < required)
-                    diskFull = true;
-            }
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
+            diskFull |= cleanupDecideWhetherDiskFull();
 
             auto msg = fmt("builder for '%s' %s",
                 yellowtxt(worker.store.printStorePath(drvPath)),
@@ -975,19 +851,12 @@ void DerivationGoal::buildDone()
         }
 
         if (buildMode == bmCheck) {
-            deleteTmpDir(true);
+            cleanupPostOutputsRegisteredModeCheck();
             done(BuildResult::Built);
             return;
         }
 
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
+        cleanupPostOutputsRegisteredModeNonCheck();
 
         /* Repeat the build if necessary. */
         if (curRound++ < nrRounds) {
@@ -1171,13 +1040,6 @@ HookReply DerivationGoal::tryBuildHook()
 }
 
 
-int childEntry(void * arg)
-{
-    ((DerivationGoal *) arg)->runChild();
-    return 1;
-}
-
-
 StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
 {
     StorePathSet paths;
@@ -1213,1769 +1075,6 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
     return paths;
 }
 
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
-    if (link(from.c_str(), to.c_str()) == -1) {
-        /* Hard-linking fails if we exceed the maximum link count on a
-           file (e.g. 32000 of ext3), which is quite possible after a
-           'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case?
-
-           It can also fail with EPERM in BeegFS v7 and earlier versions
-           which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
-            throw SysError("linking '%s' to '%s'", to, from);
-        copyPath(from, to);
-    }
-}
-
-
-void DerivationGoal::startBuilder()
-{
-    /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
-    if (drv->isBuiltin())
-        preloadNSS();
-
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
-    }
-
-    if (auto localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
-        auto & localStore = *localStoreP;
-        if (localStore.storeDir != localStore.realStoreDir) {
-            #if __linux__
-                useChroot = true;
-            #else
-                throw Error("building using a diverted store is not supported on this platform");
-            #endif
-        }
-    }
-
-    /* Create a temporary directory where the build will take
-       place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
-    chownToBuilder(tmpDir);
-
-    for (auto & [outputName, status] : initialOutputs) {
-        /* Set scratch path we'll actually use during the build.
-
-           If we're not doing a chroot build, but we have some valid
-           output paths.  Since we can't just overwrite or delete
-           them, we have to do hash rewriting: i.e. in the
-           environment/arguments passed to the build, we replace the
-           hashes of the valid outputs with unique dummy strings;
-           after the build, we discard the redirected outputs
-           corresponding to the valid outputs, and rewrite the
-           contents of the new outputs to replace the dummy strings
-           with the actual hashes. */
-        auto scratchPath =
-            !status.known
-                ? makeFallbackPath(outputName)
-            : !needsHashRewrite()
-                /* Can always use original path in sandbox */
-                ? status.known->path
-            : !status.known->isPresent()
-                /* If path doesn't yet exist can just use it */
-                ? status.known->path
-            : buildMode != bmRepair && !status.known->isValid()
-                /* If we aren't repairing we'll delete a corrupted path, so we
-                   can use original path */
-                ? status.known->path
-            :   /* If we are repairing or the path is totally valid, we'll need
-                   to use a temporary path */
-                makeFallbackPath(status.known->path);
-        scratchOutputs.insert_or_assign(outputName, scratchPath);
-
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
-        /* Substitute output placeholders with the scratch output paths.
-           We'll use during the build. */
-        inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
-        /* Additional tasks if we know the final path a priori. */
-        if (!status.known) continue;
-        auto fixedFinalPath = status.known->path;
-
-        /* Additional tasks if the final and scratch are both known and
-           differ. */
-        if (fixedFinalPath == scratchPath) continue;
-
-        /* Ensure scratch path is ours to use. */
-        deletePath(worker.store.printStorePath(scratchPath));
-
-        /* Rewrite and unrewrite paths */
-        {
-            std::string h1 { fixedFinalPath.hashPart() };
-            std::string h2 { scratchPath.hashPart() };
-            inputRewrites[h1] = h2;
-        }
-
-        redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
-    }
-
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
-    writeStructuredAttrs();
-
-    /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
-        /* The `exportReferencesGraph' feature allows the references graph
-           to be passed to a builder.  This attribute should be a list of
-           pairs [name1 path1 name2 path2 ...].  The references graph of
-           each `pathN' will be stored in a text file `nameN' in the
-           temporary build directory.  The text files have the format used
-           by `nix-store --register-validity'.  However, the deriver
-           fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
-        Strings ss = tokenizeString<Strings>(s);
-        if (ss.size() % 2 != 0)
-            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
-        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
-            if (!std::regex_match(fileName, regex))
-                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
-            auto storePathS = *i++;
-            if (!worker.store.isInStore(storePathS))
-                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
-            auto storePath = worker.store.toStorePath(storePathS).first;
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
-        }
-    }
-
-    if (useChroot) {
-
-        /* Allow a user-configurable set of directories from the
-           host file system. */
-        dirsInChroot.clear();
-
-        for (auto i : settings.sandboxPaths.get()) {
-            if (i.empty()) continue;
-            bool optional = false;
-            if (i[i.size() - 1] == '?') {
-                optional = true;
-                i.pop_back();
-            }
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = {i, optional};
-            else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
-        }
-        dirsInChroot[tmpDirInSandbox] = tmpDir;
-
-        /* Add the closure of store paths to the chroot. */
-        StorePathSet closure;
-        for (auto & i : dirsInChroot)
-            try {
-                if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
-            } catch (InvalidPath & e) {
-            } catch (Error & e) {
-                e.addTrace({}, "while processing 'sandbox-paths'");
-                throw;
-            }
-        for (auto & i : closure) {
-            auto p = worker.store.printStorePath(i);
-            dirsInChroot.insert_or_assign(p, p);
-        }
-
-        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
-        /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
-        for (auto & i : impurePaths) {
-            bool found = false;
-            /* Note: we're not resolving symlinks here to prevent
-               giving a non-root user info about inaccessible
-               files. */
-            Path canonI = canonPath(i);
-            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
-            for (auto & a : allowedPaths) {
-                Path canonA = canonPath(a);
-                if (canonI == canonA || isInDir(canonI, canonA)) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
-                    worker.store.printStorePath(drvPath), i);
-
-            dirsInChroot[i] = i;
-        }
-
-#if __linux__
-        /* Create a temporary directory in which we set up the chroot
-           environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
-
-        /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
-        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError("cannot create '%1%'", chrootRootDir);
-
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
-        /* Create a writable /tmp in the chroot.  Many builders need
-           this.  (Of course they should really respect $TMPDIR
-           instead.) */
-        Path chrootTmpDir = chrootRootDir + "/tmp";
-        createDirs(chrootTmpDir);
-        chmod_(chrootTmpDir, 01777);
-
-        /* Create a /etc/passwd with entries for the build user and the
-           nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU. */
-        createDirs(chrootRootDir + "/etc");
-
-        /* Declare the build user's group so that programs get a consistent
-           view of the system (e.g., "id -gn"). */
-        writeFile(chrootRootDir + "/etc/group",
-            fmt("root:x:0:\n"
-                "nixbld:!:%1%:\n"
-                "nogroup:x:65534:\n", sandboxGid()));
-
-        /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
-        /* Make the closure of the inputs available in the chroot,
-           rather than the whole Nix store.  This prevents any access
-           to undeclared dependencies.  Directories are bind-mounted,
-           while other inputs are hard-linked (since only directories
-           can be bind-mounted).  !!! As an extra security
-           precaution, make the fake Nix store only writable by the
-           build user. */
-        Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-        createDirs(chrootStoreDir);
-        chmod_(chrootStoreDir, 01775);
-
-        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
-        for (auto & i : inputPaths) {
-            auto p = worker.store.printStorePath(i);
-            Path r = worker.store.toRealPath(p);
-            if (S_ISDIR(lstat(r).st_mode))
-                dirsInChroot.insert_or_assign(p, r);
-            else
-                linkOrCopy(r, chrootRootDir + p);
-        }
-
-        /* If we're repairing, checking or rebuilding part of a
-           multiple-outputs derivation, it's possible that we're
-           rebuilding a path that is in settings.dirsInChroot
-           (typically the dependencies of /bin/sh).  Throw them
-           out. */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
-               should be fresh.  Freshness means it is impossible that the path
-               is already in the sandbox, so we don't need to worry about
-               removing it.  */
-            if (i.second.second)
-                dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
-        }
-
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
-#else
-        throw Error("sandboxing builds is not supported on this platform");
-#endif
-    }
-
-    if (needsHashRewrite() && pathExists(homeDir))
-        throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
-    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
-            % settings.preBuildHook);
-        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
-            Strings({ worker.store.printStorePath(drvPath) });
-        enum BuildHookState {
-            stBegin,
-            stExtraChrootDirs
-        };
-        auto state = stBegin;
-        auto lines = runProgram(settings.preBuildHook, false, args);
-        auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
-                nlPos = lines.find('\n', lastPos)) {
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
-            lastPos = nlPos + 1;
-            if (state == stBegin) {
-                if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
-                    state = stExtraChrootDirs;
-                } else {
-                    throw Error("unknown pre-build hook command '%1%'", line);
-                }
-            } else if (state == stExtraChrootDirs) {
-                if (line == "") {
-                    state = stBegin;
-                } else {
-                    auto p = line.find('=');
-                    if (p == string::npos)
-                        dirsInChroot[line] = line;
-                    else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
-                }
-            }
-        }
-    }
-
-    /* Fire up a Nix daemon to process recursive Nix calls from the
-       builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
-        startDaemon();
-
-    /* Run the builder. */
-    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
-    /* Create the log file. */
-    Path logFile = openLogFile();
-
-    /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
-
-    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
-    if (!builderOut.readSide)
-        throw SysError("opening pseudoterminal master");
-
-    std::string slaveName(ptsname(builderOut.readSide.get()));
-
-    if (buildUser) {
-        if (chmod(slaveName.c_str(), 0600))
-            throw SysError("changing mode of pseudoterminal slave");
-
-        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
-            throw SysError("changing owner of pseudoterminal slave");
-    }
-#if __APPLE__
-    else {
-        if (grantpt(builderOut.readSide.get()))
-            throw SysError("granting access to pseudoterminal slave");
-    }
-#endif
-
-    #if 0
-    // Mount the pt in the sandbox so that the "tty" command works.
-    // FIXME: this doesn't work with the new devpts in the sandbox.
-    if (useChroot)
-        dirsInChroot[slaveName] = {slaveName, false};
-    #endif
-
-    if (unlockpt(builderOut.readSide.get()))
-        throw SysError("unlocking pseudoterminal");
-
-    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
-    if (!builderOut.writeSide)
-        throw SysError("opening pseudoterminal slave");
-
-    // Put the pt into raw mode to prevent \n -> \r\n translation.
-    struct termios term;
-    if (tcgetattr(builderOut.writeSide.get(), &term))
-        throw SysError("getting pseudoterminal attributes");
-
-    cfmakeraw(&term);
-
-    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
-        throw SysError("putting pseudoterminal into raw mode");
-
-    result.startTime = time(0);
-
-    /* Fork a child to build the package. */
-    ProcessOptions options;
-
-#if __linux__
-    if (useChroot) {
-        /* Set up private namespaces for the build:
-
-           - The PID namespace causes the build to start as PID 1.
-             Processes outside of the chroot are not visible to those
-             on the inside, but processes inside the chroot are
-             visible from the outside (though with different PIDs).
-
-           - The private mount namespace ensures that all the bind
-             mounts we do will only show up in this process and its
-             children, and will disappear automatically when we're
-             done.
-
-           - The private network namespace ensures that the builder
-             cannot talk to the outside world (or vice versa).  It
-             only has a private loopback interface. (Fixed-output
-             derivations are not run in a private network namespace
-             to allow functions like fetchurl to work.)
-
-           - The IPC namespace prevents the builder from communicating
-             with outside processes using SysV IPC mechanisms (shared
-             memory, message queues, semaphores).  It also ensures
-             that all IPC objects are destroyed when the builder
-             exits.
-
-           - The UTS namespace ensures that builders see a hostname of
-             localhost rather than the actual hostname.
-
-           We use a helper process to do the clone() to work around
-           clone() being broken in multi-threaded programs due to
-           at-fork handlers not being run. Note that we use
-           CLONE_PARENT to ensure that the real builder is parented to
-           us.
-        */
-
-        if (!(derivationIsImpure(derivationType)))
-            privateNetwork = true;
-
-        userNamespaceSync.create();
-
-        options.allowVfork = false;
-
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
-
-        Pid helper = startProcess([&]() {
-
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace.  FIXME:
-               this means that if we're not root in the parent
-               namespace, we can't drop additional groups; they will
-               be mapped to nogroup in the child namespace. There does
-               not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)
-               See also https://lwn.net/Articles/621612/. */
-            if (getuid() == 0 && setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-
-            size_t stackSize = 1 * 1024 * 1024;
-            char * stack = (char *) mmap(0, stackSize,
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                flags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
-
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            /* Otherwise exit with EPERM so we can handle this in the
-               parent. This is only done when sandbox-fallback is set
-               to true (the default). */
-            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(1);
-            if (child == -1) throw SysError("cloning builder process");
-
-            writeFull(builderOut.writeSide.get(),
-                fmt("%d %d\n", usingUserNamespace, child));
-            _exit(0);
-        }, options);
-
-        int res = helper.wait();
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
-            throw Error("unable to start build process");
-
-        userNamespaceSync.readSide = -1;
-
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() {
-            userNamespaceSync.writeSide = -1;
-        });
-
-        auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
-        assert(ss.size() == 2);
-        usingUserNamespace = ss[0] == "1";
-        pid = string2Int<pid_t>(ss[1]).value();
-
-        if (usingUserNamespace) {
-            /* Set the UID/GID mapping of the builder's user namespace
-               such that the sandbox user maps to the build user, or to
-               the calling user (if build users are disabled). */
-            uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
-            uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-
-            writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
-
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
-            writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
-        } else {
-            debug("note: not using a user namespace");
-            if (!buildUser)
-                throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
-        }
-
-        /* Now that we now the sandbox uid, we can write
-           /etc/passwd. */
-        writeFile(chrootRootDir + "/etc/passwd", fmt(
-                "root:x:0:0:Nix build user:%3%:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
-        /* Save the mount namespace of the child. We have to do this
-           *before* the child does a chroot. */
-        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
-        if (sandboxMountNamespace.get() == -1)
-            throw SysError("getting sandbox mount namespace");
-
-        /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
-
-    } else
-#endif
-    {
-    fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
-        pid = startProcess([&]() {
-            runChild();
-        }, options);
-    }
-
-    /* parent */
-    pid.setSeparatePG(true);
-    builderOut.writeSide = -1;
-    worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
-    /* Check if setting up the build environment failed. */
-    std::vector<std::string> msgs;
-    while (true) {
-        string msg = [&]() {
-            try {
-                return readLine(builderOut.readSide.get());
-            } catch (Error & e) {
-                e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
-                    concatStringsSep("|", msgs));
-                throw e;
-            }
-        }();
-        if (string(msg, 0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
-            FdSource source(builderOut.readSide.get());
-            auto ex = readError(source);
-            ex.addTrace({}, "while setting up the build environment");
-            throw ex;
-        }
-        debug("sandbox setup: " + msg);
-        msgs.push_back(std::move(msg));
-    }
-}
-
-
-void DerivationGoal::initTmpDir() {
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
-
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
-                env[i.first] = i.second;
-            } else {
-                auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
-                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-            }
-        }
-
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
-    initTmpDir();
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i).value_or("");
-    }
-
-    /* Currently structured log messages piggyback on stderr, but we
-       may change that in the future. So tell the builder which file
-       descriptor to use for that. */
-    env["NIX_LOG_FD"] = "2";
-
-    /* Trigger colored output in various tools. */
-    env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
-
-    auto json = *structuredAttrs;
-
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
-    }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : virtual LocalFSStoreConfig
-{
-    using LocalFSStoreConfig::LocalFSStoreConfig;
-    const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
-   paths that are in the input closures of the build or were added via
-   recursive Nix calls. */
-struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
-{
-    ref<LocalStore> next;
-
-    DerivationGoal & goal;
-
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params)
-        , LocalFSStoreConfig(params)
-        , RestrictedStoreConfig(params)
-        , Store(params)
-        , LocalFSStore(params)
-        , next(next), goal(goal)
-    { }
-
-    Path getRealStoreDir() override
-    { return next->realStoreDir; }
-
-    std::string getUri() override
-    { return next->getUri(); }
-
-    StorePathSet queryAllValidPaths() override
-    {
-        StorePathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
-        return paths;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        if (goal.isAllowed(path)) {
-            try {
-                /* Censor impure information. */
-                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.reset();
-                info->registrationTime = 0;
-                info->ultimate = false;
-                info->sigs.clear();
-                callback(info);
-            } catch (InvalidPath &) {
-                callback(nullptr);
-            }
-        } else
-            callback(nullptr);
-    };
-
-    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
-    { }
-
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
-        return next->queryPartialDerivationOutputMap(path);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { throw Error("queryPathFromHashPart"); }
-
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore"); }
-
-    void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
-    {
-        next->addToStore(info, narSource, repair, checkSigs);
-        goal.addDependency(info.path);
-    }
-
-    StorePath addTextToStore(const string & name, const string & s,
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
-        LocalFSStore::narFromPath(path, sink);
-    }
-
-    void ensurePath(const StorePath & path) override
-    {
-        if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
-        /* Nothing to be done; 'path' must already be valid. */
-    }
-
-    void registerDrvOutput(const Realisation & info) override
-    // XXX: This should probably be allowed as a no-op if the realisation
-    // corresponds to an allowed derivation
-    { throw Error("registerDrvOutput"); }
-
-    std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
-    // XXX: This should probably be allowed if the realisation corresponds to
-    // an allowed derivation
-    { throw Error("queryRealisation"); }
-
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
-    {
-        if (buildMode != bmNormal) throw Error("unsupported build mode");
-
-        StorePathSet newPaths;
-
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
-        }
-
-        next->buildPaths(paths, buildMode);
-
-        for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
-        }
-
-        StorePathSet closure;
-        next->computeFSClosure(newPaths, closure);
-        for (auto & path : closure)
-            goal.addDependency(path);
-    }
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { unsupported("buildDerivation"); }
-
-    void addTempRoot(const StorePath & path) override
-    { }
-
-    void addIndirectRoot(const Path & path) override
-    { }
-
-    Roots findRoots(bool censor) override
-    { return Roots(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { }
-
-    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
-    { unsupported("addSignatures"); }
-
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
-        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        uint64_t & downloadSize, uint64_t & narSize) override
-    {
-        /* This is slightly impure since it leaks information to the
-           client about what paths will be built/substituted or are
-           already present. Probably not a big deal. */
-
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
-            else
-                unknown.insert(path.path);
-        }
-
-        next->queryMissing(allowed, willBuild, willSubstitute,
-            unknown, downloadSize, narSize);
-    }
-};
-
-
-void DerivationGoal::startDaemon()
-{
-    settings.requireExperimentalFeature("recursive-nix");
-
-    Store::Params params;
-    params["path-info-cache-size"] = "0";
-    params["store"] = worker.store.storeDir;
-    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
-        params["root"] = localStore->rootDir;
-    params["state"] = "/no-such-path";
-    params["log"] = "/no-such-path";
-    auto store = make_ref<RestrictedStore>(params,
-        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
-        *this);
-
-    addedPaths.clear();
-
-    auto socketName = ".nix-socket";
-    Path socketPath = tmpDir + "/" + socketName;
-    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
-    daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
-    chownToBuilder(socketPath);
-
-    daemonThread = std::thread([this, store]() {
-
-        while (true) {
-
-            /* Accept a connection. */
-            struct sockaddr_un remoteAddr;
-            socklen_t remoteAddrLen = sizeof(remoteAddr);
-
-            AutoCloseFD remote = accept(daemonSocket.get(),
-                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
-            if (!remote) {
-                if (errno == EINTR) continue;
-                if (errno == EINVAL) break;
-                throw SysError("accepting connection");
-            }
-
-            closeOnExec(remote.get());
-
-            debug("received daemon connection");
-
-            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
-                FdSource from(remote.get());
-                FdSink to(remote.get());
-                try {
-                    daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
-                        [&](Store & store) { store.createUser("nobody", 65535); });
-                    debug("terminated daemon connection");
-                } catch (SysError &) {
-                    ignoreException();
-                }
-            });
-
-            daemonWorkerThreads.push_back(std::move(workerThread));
-        }
-
-        debug("daemon shutting down");
-    });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
-
-    if (daemonThread.joinable())
-        daemonThread.join();
-
-    // FIXME: should prune worker threads more quickly.
-    // FIXME: shutdown the client socket to speed up worker termination.
-    for (auto & thread : daemonWorkerThreads)
-        thread.join();
-    daemonWorkerThreads.clear();
-
-    daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
-    if (isAllowed(path)) return;
-
-    addedPaths.insert(path);
-
-    /* If we're doing a sandbox build, then we have to make the path
-       appear in the sandbox. */
-    if (useChroot) {
-
-        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
-        #if __linux__
-
-            Path source = worker.store.Store::toRealPath(path);
-            Path target = chrootRootDir + worker.store.printStorePath(path);
-            debug("bind-mounting %s -> %s", target, source);
-
-            if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
-            auto st = lstat(source);
-
-            if (S_ISDIR(st.st_mode)) {
-
-                /* Bind-mount the path into the sandbox. This requires
-                   entering its mount namespace, which is not possible
-                   in multithreaded programs. So we do this in a
-                   child process.*/
-                Pid child(startProcess([&]() {
-
-                    if (setns(sandboxMountNamespace.get(), 0) == -1)
-                        throw SysError("entering sandbox mount namespace");
-
-                    createDirs(target);
-
-                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
-                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
-                    _exit(0);
-                }));
-
-                int status = child.wait();
-                if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
-            } else
-                linkOrCopy(source, target);
-
-        #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
-                worker.store.printStorePath(path));
-        #endif
-
-    }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
-    if (!buildUser) return;
-    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
-    if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
-    scmp_filter_ctx ctx;
-
-    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
-        throw SysError("unable to initialize seccomp mode 2");
-
-    Finally cleanup([&]() {
-        seccomp_release(ctx);
-    });
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
-        throw SysError("unable to add 32-bit seccomp architecture");
-
-    if (nativeSystem == "x86_64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
-        throw SysError("unable to add X32 seccomp architecture");
-
-    if (nativeSystem == "aarch64-linux" &&
-        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
-    /* Prevent builders from creating setuid/setgid binaries. */
-    for (int perm : { S_ISUID, S_ISGID }) {
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-
-        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
-            throw SysError("unable to add seccomp rule");
-    }
-
-    /* Prevent builders from creating EAs or ACLs. Not all filesystems
-       support these, and they're not allowed in the Nix store because
-       they're not representable in the NAR serialisation. */
-    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
-        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
-        throw SysError("unable to add seccomp rule");
-
-    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
-        throw SysError("unable to set 'no new privileges' seccomp attribute");
-
-    if (seccomp_load(ctx) != 0)
-        throw SysError("unable to load seccomp BPF program");
-#else
-    throw Error(
-        "seccomp is not supported on this platform; "
-        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
-    /* Warning: in the child we should absolutely not make any SQLite
-       calls! */
-
-    try { /* child */
-
-        commonChildInit(builderOut);
-
-        try {
-            setupSeccomp();
-        } catch (...) {
-            if (buildUser) throw;
-        }
-
-        bool setUser = true;
-
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
-        std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
-
-#if __linux__
-        if (useChroot) {
-
-            userNamespaceSync.writeSide = -1;
-
-            if (drainFD(userNamespaceSync.readSide.get()) != "1")
-                throw Error("user namespace initialisation failed");
-
-            userNamespaceSync.readSide = -1;
-
-            if (privateNetwork) {
-
-                /* Initialise the loopback interface. */
-                AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
-                if (!fd) throw SysError("cannot open IP socket");
-
-                struct ifreq ifr;
-                strcpy(ifr.ifr_name, "lo");
-                ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
-                if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
-                    throw SysError("cannot set loopback interface flags");
-            }
-
-            /* Set the hostname etc. to fixed values. */
-            char hostname[] = "localhost";
-            if (sethostname(hostname, sizeof(hostname)) == -1)
-                throw SysError("cannot set host name");
-            char domainname[] = "(none)"; // kernel default
-            if (setdomainname(domainname, sizeof(domainname)) == -1)
-                throw SysError("cannot set domain name");
-
-            /* Make all filesystems private.  This is necessary
-               because subtrees may have been mounted as "shared"
-               (MS_SHARED).  (Systemd does this, for instance.)  Even
-               though we have a private mount namespace, mounting
-               filesystems on top of a shared subtree still propagates
-               outside of the namespace.  Making a subtree private is
-               local to the namespace, though, so setting MS_PRIVATE
-               does not affect the outside world. */
-            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
-                throw SysError("unable to make '/' private");
-
-            /* Bind-mount chroot directory to itself, to treat it as a
-               different filesystem from /, as needed for pivot_root. */
-            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
-            /* Bind-mount the sandbox's Nix store onto itself so that
-               we can mark it as a "shared" subtree, allowing bind
-               mounts made in *this* mount namespace to be propagated
-               into the child namespace created by the
-               unshare(CLONE_NEWNS) call below.
-
-               Marking chrootRootDir as MS_SHARED causes pivot_root()
-               to fail with EINVAL. Don't know why. */
-            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
-            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
-            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
-                throw SysError("unable to make '%s' shared", chrootStoreDir);
-
-            /* Set up a nearly empty /dev, unless the user asked to
-               bind-mount the host /dev. */
-            Strings ss;
-            if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
-                createDirs(chrootRootDir + "/dev/shm");
-                createDirs(chrootRootDir + "/dev/pts");
-                ss.push_back("/dev/full");
-                if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
-                    ss.push_back("/dev/kvm");
-                ss.push_back("/dev/null");
-                ss.push_back("/dev/random");
-                ss.push_back("/dev/tty");
-                ss.push_back("/dev/urandom");
-                ss.push_back("/dev/zero");
-                createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
-                createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
-                createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
-                createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
-            }
-
-            /* Fixed-output derivations typically need to access the
-               network, so give them access to /etc/resolv.conf and so
-               on. */
-            if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
-                // Only use nss functions to resolve hosts and
-                // services. Don’t use it for anything else that may
-                // be configured for this system. This limits the
-                // potential impurities introduced in fixed-outputs.
-                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
-            }
-
-            for (auto & i : ss) dirsInChroot.emplace(i, i);
-
-            /* Bind-mount all the directories from the "host"
-               filesystem that we want in the chroot
-               environment. */
-            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug("bind mounting '%1%' to '%2%'", source, target);
-                struct stat st;
-                if (stat(source.c_str(), &st) == -1) {
-                    if (optional && errno == ENOENT)
-                        return;
-                    else
-                        throw SysError("getting attributes of path '%1%'", source);
-                }
-                if (S_ISDIR(st.st_mode))
-                    createDirs(target);
-                else {
-                    createDirs(dirOf(target));
-                    writeFile(target, "");
-                }
-                if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
-            };
-
-            for (auto & i : dirsInChroot) {
-                if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
-            }
-
-            /* Bind a new instance of procfs on /proc. */
-            createDirs(chrootRootDir + "/proc");
-            if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
-                throw SysError("mounting /proc");
-
-            /* Mount a new tmpfs on /dev/shm to ensure that whatever
-               the builder puts in /dev/shm is cleaned up automatically. */
-            if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
-                throw SysError("mounting /dev/shm");
-
-            /* Mount a new devpts on /dev/pts.  Note that this
-               requires the kernel to be compiled with
-               CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
-               if /dev/ptx/ptmx exists). */
-            if (pathExists("/dev/pts/ptmx") &&
-                !pathExists(chrootRootDir + "/dev/ptmx")
-                && !dirsInChroot.count("/dev/pts"))
-            {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
-                {
-                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
-                    /* Make sure /dev/pts/ptmx is world-writable.  With some
-                       Linux versions, it is created with permissions 0.  */
-                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
-                } else {
-                    if (errno != EINVAL)
-                        throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
-                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
-                }
-            }
-
-            /* Unshare this mount namespace. This is necessary because
-               pivot_root() below changes the root of the mount
-               namespace. This means that the call to setns() in
-               addDependency() would hide the host's filesystem,
-               making it impossible to bind-mount paths from the host
-               Nix store into the sandbox. Therefore, we save the
-               pre-pivot_root namespace in
-               sandboxMountNamespace. Since we made /nix/store a
-               shared subtree above, this allows addDependency() to
-               make paths appear in the sandbox. */
-            if (unshare(CLONE_NEWNS) == -1)
-                throw SysError("unsharing mount namespace");
-
-            /* Do the chroot(). */
-            if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
-            if (mkdir("real-root", 0) == -1)
-                throw SysError("cannot create real-root directory");
-
-            if (pivot_root(".", "real-root") == -1)
-                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
-            if (chroot(".") == -1)
-                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
-            if (umount2("real-root", MNT_DETACH) == -1)
-                throw SysError("cannot unmount real root filesystem");
-
-            if (rmdir("real-root") == -1)
-                throw SysError("cannot remove real-root directory");
-
-            /* Switch to the sandbox uid/gid in the user namespace,
-               which corresponds to the build user or calling user in
-               the parent namespace. */
-            if (setgid(sandboxGid()) == -1)
-                throw SysError("setgid failed");
-            if (setuid(sandboxUid()) == -1)
-                throw SysError("setuid failed");
-
-            setUser = false;
-        }
-#endif
-
-        if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError("changing into '%1%'", tmpDir);
-
-        /* Close all other file descriptors. */
-        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
-        /* Disable core dumps by default. */
-        struct rlimit limit = { 0, RLIM_INFINITY };
-        setrlimit(RLIMIT_CORE, &limit);
-
-        // FIXME: set other limits to deterministic values?
-
-        /* Fill in the environment. */
-        Strings envStrs;
-        for (auto & i : env)
-            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
-        /* If we are running in `build-users' mode, then switch to the
-           user we allocated above.  Make sure that we drop all root
-           privileges.  Note that above we have closed all file
-           descriptors except std*, so that's safe.  Also note that
-           setuid() when run as root sets the real, effective and
-           saved UIDs. */
-        if (setUser && buildUser) {
-            /* Preserve supplementary groups of the build user, to allow
-               admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
-                throw SysError("cannot set supplementary groups of build user");
-
-            if (setgid(buildUser->getGID()) == -1 ||
-                getgid() != buildUser->getGID() ||
-                getegid() != buildUser->getGID())
-                throw SysError("setgid failed");
-
-            if (setuid(buildUser->getUID()) == -1 ||
-                getuid() != buildUser->getUID() ||
-                geteuid() != buildUser->getUID())
-                throw SysError("setuid failed");
-        }
-
-        /* Fill in the arguments. */
-        Strings args;
-
-        const char *builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
-#if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
-
-            if (useChroot) {
-
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
-
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : dirsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
-                while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
-                    cur = dirOf(cur);
-                }
-
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    dirsInChroot[p] = p;
-                }
-
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
-
-                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
-                if (derivationIsImpure(derivationType))
-                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : dirsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    string path = i.first;
-                    struct stat st;
-                    if (lstat(path.c_str(), &st)) {
-                        if (i.second.optional && errno == ENOENT)
-                            continue;
-                        throw SysError("getting attributes of path '%s", path);
-                    }
-                    if (S_ISDIR(st.st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                args.push_back("-D");
-                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
-            } else {
-                builder = drv->builder.c_str();
-                args.push_back(std::string(baseNameOf(drv->builder)));
-            }
-        }
-#else
-        else {
-            builder = drv->builder.c_str();
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
-#endif
-
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
-
-        /* Execute the program.  This should not return. */
-        if (drv->isBuiltin()) {
-            try {
-                logger = makeJSONLogger(*logger);
-
-                BasicDerivation & drv2(*drv);
-                for (auto & e : drv2.env)
-                    e.second = rewriteStrings(e.second, inputRewrites);
-
-                if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(drv2, netrcData);
-                else if (drv->builder == "builtin:buildenv")
-                    builtinBuildenv(drv2);
-                else if (drv->builder == "builtin:unpack-channel")
-                    builtinUnpackChannel(drv2);
-                else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
-                _exit(0);
-            } catch (std::exception & e) {
-                writeFull(STDERR_FILENO, e.what() + std::string("\n"));
-                _exit(1);
-            }
-        }
-
-#if __APPLE__
-        posix_spawnattr_t attrp;
-
-        if (posix_spawnattr_init(&attrp))
-            throw SysError("failed to initialize builder");
-
-        if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
-            throw SysError("failed to initialize builder");
-
-        if (drv->platform == "aarch64-darwin") {
-            // Unset kern.curproc_arch_affinity so we can escape Rosetta
-            int affinity = 0;
-            sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
-
-            cpu_type_t cpu = CPU_TYPE_ARM64;
-            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
-        } else if (drv->platform == "x86_64-darwin") {
-            cpu_type_t cpu = CPU_TYPE_X86_64;
-            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
-        }
-
-        posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-#else
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-#endif
-
-        throw SysError("executing '%1%'", drv->builder);
-
-    } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
-        _exit(1);
-    }
-}
-
 
 void DerivationGoal::registerOutputs()
 {
@@ -2986,698 +1085,23 @@ void DerivationGoal::registerOutputs()
        We can only early return when the outputs are known a priori. For
        floating content-addressed derivations this isn't the case.
      */
-    if (hook) {
-        bool allValid = true;
-        for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
-            if (!outputPath || !worker.store.isValidPath(*outputPath))
-                allValid = false;
-            else
-                finalOutputs.insert_or_assign(outputName, *outputPath);
-        }
-        if (allValid) return;
-    }
-
-    std::map<std::string, ValidPathInfo> infos;
-
-    /* Set of inodes seen during calls to canonicalisePathMetaData()
-       for this build's outputs.  This needs to be shared between
-       outputs to allow hard links between outputs. */
-    InodesSeen inodesSeen;
-
-    Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
-    std::exception_ptr delayedException;
-
-    /* The paths that can be referenced are the input closures, the
-       output paths, and any paths that have been built via recursive
-       Nix calls. */
-    StorePathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
-
-    /* FIXME `needsHashRewrite` should probably be removed and we get to the
-       real reason why we aren't using the chroot dir */
-    auto toRealPathChroot = [&](const Path & p) -> Path {
-        return useChroot && !needsHashRewrite()
-            ? chrootRootDir + p
-            : worker.store.toRealPath(p);
-    };
-
-    /* Check whether the output paths were created, and make all
-       output paths read-only.  Then get the references of each output (that we
-       might need to register), so we can topologically sort them. For the ones
-       that are most definitely already installed, we just store their final
-       name so we can also use it in rewrites. */
-    StringSet outputsToSort;
-    struct AlreadyRegistered { StorePath path; };
-    struct PerhapsNeedToRegister { StorePathSet refs; };
-    std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
-    std::map<std::string, struct stat> outputStats;
-    for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
-        outputsToSort.insert(outputName);
-
-        /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
-
-        /* Don't register if already valid, and not checking */
-        initialInfo.wanted = buildMode == bmCheck
-            || !(initialInfo.known && initialInfo.known->isValid());
-        if (!initialInfo.wanted) {
-            outputReferencesIfUnregistered.insert_or_assign(
-                outputName,
-                AlreadyRegistered { .path = initialInfo.known->path });
+    for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
+        if (!wantOutput(outputName, wantedOutputs))
             continue;
-        }
-
-        struct stat st;
-        if (lstat(actualPath.c_str(), &st) == -1) {
-            if (errno == ENOENT)
-                throw BuildError(
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
-
-#ifndef __CYGWIN__
-        /* Check that the output is not group or world writable, as
-           that means that someone else can have interfered with the
-           build.  Also, the output should be owned by the build
-           user. */
-        if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser && st.st_uid != buildUser->getUID()))
+        if (!optOutputPath)
             throw BuildError(
-                    "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
-                    actualPath, outputName);
-#endif
+                "output '%s' from derivation '%s' does not have a known output path",
+                outputName, worker.store.printStorePath(drvPath));
+        auto & outputPath = *optOutputPath;
+        if (!worker.store.isValidPath(outputPath))
+            throw BuildError(
+                "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
+                outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
 
-        /* Canonicalise first.  This ensures that the path we're
-           rewriting doesn't contain a hard link to /etc/shadow or
-           something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
-
-        debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
-        /* Pass blank Sink as we are not ready to hash data at this stage. */
-        NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
-        outputReferencesIfUnregistered.insert_or_assign(
-            outputName,
-            PerhapsNeedToRegister { .refs = references });
-        outputStats.insert_or_assign(outputName, std::move(st));
-    }
-
-    auto sortedOutputNames = topoSort(outputsToSort,
-        {[&](const std::string & name) {
-            return std::visit(overloaded {
-                /* Since we'll use the already installed versions of these, we
-                   can treat them as leaves and ignore any references they
-                   have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
-                    StringSet referencedOutputs;
-                    /* FIXME build inverted map up front so no quadratic waste here */
-                    for (auto & r : refs.refs)
-                        for (auto & [o, p] : scratchOutputs)
-                            if (r == p)
-                                referencedOutputs.insert(o);
-                    return referencedOutputs;
-                },
-            }, outputReferencesIfUnregistered.at(name));
-        }},
-        {[&](const std::string & path, const std::string & parent) {
-            // TODO with more -vvvv also show the temporary paths for manual inspection.
-            return BuildError(
-                "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
-                worker.store.printStorePath(drvPath), path, parent);
-        }});
-
-    std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
-    for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
-        auto finish = [&](StorePath finalStorePath) {
-            /* Store the final path */
-            finalOutputs.insert_or_assign(outputName, finalStorePath);
-            /* The rewrite rule will be used in downstream outputs that refer to
-               use. This is why the topological sort is essential to do first
-               before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
-        };
-
-        std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
-                finish(skippedFinalPath.path);
-                return std::nullopt;
-            },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
-                return r.refs;
-            },
-        }, outputReferencesIfUnregistered.at(outputName));
-
-        if (!referencesOpt)
-            continue;
-        auto references = *referencesOpt;
-
-        auto rewriteOutput = [&]() {
-            /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
-                warn("rewriting hashes in '%1%'; cross fingers", actualPath);
-
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
-                deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
-                restorePath(actualPath, source);
-
-                /* FIXME: set proper permissions in restorePath() so
-                   we don't have to do another traversal. */
-                canonicalisePathMetaData(actualPath, -1, inodesSeen);
-            }
-        };
-
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
-            /* In the CA case, we need the rewritten refs to calculate the
-               final path, therefore we look for a *non-rewritten
-               self-reference, and use a bool rather try to solve the
-               computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
-            };
-            for (auto & r : references) {
-                auto name = r.name();
-                auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
-                    res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
-                    newRef += '-';
-                    newRef += name;
-                    res.second.insert(StorePath { newRef });
-                }
-            }
-            return res;
-        };
-
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
-            if (outputHash.method == FileIngestionMethod::Flat) {
-                /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
-                    throw BuildError(
-                        "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
-                        actualPath);
-            }
-            rewriteOutput();
-            /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
-            auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
-            HashModuloSink narSink { htSHA256, oldHashPart };
-            dumpPath(actualPath, narSink);
-            auto narHashAndSize = narSink.finish();
-            ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
-                narHashAndSize.first,
-            };
-            newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
-            if (scratchPath != newInfo0.path) {
-                // Also rewrite the output path
-                auto source = sinkToSource([&](Sink & nextSink) {
-                    StringSink sink;
-                    dumpPath(actualPath, sink);
-                    RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
-                    rsink2(*sink.s);
-                    rsink2.flush();
-                });
-                Path tmpPath = actualPath + ".tmp";
-                restorePath(tmpPath, *source);
-                deletePath(actualPath);
-                movePath(tmpPath, actualPath);
-            }
-
-            assert(newInfo0.ca);
-            return newInfo0;
-        };
-
-        ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
-                /* input-addressed case */
-                auto requiredFinalPath = output.path;
-                /* Preemptively add rewrite rule for final hash, as that is
-                   what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
-                    outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
-                        std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
-                ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
-                newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFixed dof) {
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
-                });
-
-                /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
-                assert(newInfo0.ca);
-                auto got = getContentAddressHash(*newInfo0.ca);
-                if (wanted != got) {
-                    /* Throw an error after registering the path as
-                       valid. */
-                    worker.hashMismatch = true;
-                    delayedException = std::make_exception_ptr(
-                        BuildError("hash mismatch in fixed-output derivation '%s':\n  specified: %s\n     got:    %s",
-                            worker.store.printStorePath(drvPath),
-                            wanted.to_string(SRI, true),
-                            got.to_string(SRI, true)));
-                }
-                return newInfo0;
-            },
-            [&](DerivationOutputCAFloating dof) {
-                return newInfoFromCA(dof);
-            },
-                [&](DerivationOutputDeferred) {
-                // No derivation should reach that point without having been
-                // rewritten first
-                assert(false);
-                // Ugly, but the compiler insists on having this return a value
-                // of type `ValidPathInfo` despite the `assert(false)`, so
-                // let's provide it
-                return *(ValidPathInfo*)0;
-            },
-        }, output.output);
-
-        /* Calculate where we'll move the output files. In the checking case we
-           will leave leave them where they are, for now, rather than move to
-           their usual "final destination" */
-        auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
-        /* Lock final output path, if not already locked. This happens with
-           floating CA derivations and hash-mismatching fixed-output
-           derivations. */
-        PathLocks dynamicOutputLock;
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
-        if (!optFixedPath ||
-            worker.store.printStorePath(*optFixedPath) != finalDestPath)
-        {
-            assert(newInfo.ca);
-            dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
-        }
-
-        /* Move files, if needed */
-        if (worker.store.toRealPath(finalDestPath) != actualPath) {
-            if (buildMode == bmRepair) {
-                /* Path already exists, need to replace it */
-                replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
-                actualPath = worker.store.toRealPath(finalDestPath);
-            } else if (buildMode == bmCheck) {
-                /* Path already exists, and we want to compare, so we leave out
-                   new path in place. */
-            } else if (worker.store.isValidPath(newInfo.path)) {
-                /* Path already exists because CA path produced by something
-                   else. No moving needed. */
-                assert(newInfo.ca);
-            } else {
-                auto destPath = worker.store.toRealPath(finalDestPath);
-                movePath(actualPath, destPath);
-                actualPath = destPath;
-            }
-        }
-
-        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
-        if (!localStoreP)
-            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
-        auto & localStore = *localStoreP;
-
-        if (buildMode == bmCheck) {
-
-            if (!worker.store.isValidPath(newInfo.path)) continue;
-            ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
-            if (newInfo.narHash != oldInfo.narHash) {
-                worker.checkMismatch = true;
-                if (settings.runDiffHook || settings.keepFailed) {
-                    auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
-                    deletePath(dst);
-                    movePath(actualPath, dst);
-
-                    handleDiffHook(
-                        buildUser ? buildUser->getUID() : getuid(),
-                        buildUser ? buildUser->getGID() : getgid(),
-                        finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
-                } else
-                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
-            }
-
-            /* Since we verified the build, it's now ultimately trusted. */
-            if (!oldInfo.ultimate) {
-                oldInfo.ultimate = true;
-                localStore.signPathInfo(oldInfo);
-                localStore.registerValidPaths({{oldInfo.path, oldInfo}});
-            }
-
-            continue;
-        }
-
-        /* For debugging, print out the referenced and unreferenced paths. */
-        for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
-                debug("referenced input: '%1%'", worker.store.printStorePath(i));
-        }
-
-        if (curRound == nrRounds) {
-            localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
-
-        newInfo.deriver = drvPath;
-        newInfo.ultimate = true;
-        localStore.signPathInfo(newInfo);
-
-        finish(newInfo.path);
-
-        /* If it's a CA path, register it right away. This is necessary if it
-           isn't statically known so that we can safely unlock the path before
-           the next iteration */
-        if (newInfo.ca)
-            localStore.registerValidPaths({{newInfo.path, newInfo}});
-
-        infos.emplace(outputName, std::move(newInfo));
-    }
-
-    if (buildMode == bmCheck) return;
-
-    /* Apply output checks. */
-    checkOutputs(infos);
-
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                result.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                printError(hint);
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
-                throw SysError("renaming '%s' to '%s'", path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return;
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  If there are cycles in the
-       outputs, this will fail. */
-    {
-        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
-        if (!localStoreP)
-            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
-        auto & localStore = *localStoreP;
-
-        ValidPathInfos infos2;
-        for (auto & [outputName, newInfo] : infos) {
-            infos2.insert_or_assign(newInfo.path, newInfo);
-        }
-        localStore.registerValidPaths(infos2);
-    }
-
-    /* In case of a fixed-output derivation hash mismatch, throw an
-       exception now that we have registered the output as valid. */
-    if (delayedException)
-        std::rethrow_exception(delayedException);
-
-    /* If we made it this far, we are sure the output matches the derivation
-       (since the delayedException would be a fixed output CA mismatch). That
-       means it's safe to link the derivation to the output hash. We must do
-       that for floating CA derivations, which otherwise couldn't be cached,
-       but it's fine to do in all cases. */
-
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-        for (auto& [outputName, newInfo] : infos)
-            worker.store.registerDrvOutput(Realisation{
-                .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName},
-                .outPath = newInfo.path});
+        finalOutputs.insert_or_assign(outputName, outputPath);
     }
 }
 
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
-    std::map<Path, const ValidPathInfo &> outputsByPath;
-    for (auto & output : outputs)
-        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
-    for (auto & output : outputs) {
-        auto & outputName = output.first;
-        auto & info = output.second;
-
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
-        /* Compute the closure and closure size of some output. This
-           is slightly tricky because some of its references (namely
-           other outputs) may not be valid yet. */
-        auto getClosure = [&](const StorePath & path)
-        {
-            uint64_t closureSize = 0;
-            StorePathSet pathsDone;
-            std::queue<StorePath> pathsLeft;
-            pathsLeft.push(path);
-
-            while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
-                pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
-
-                auto i = outputsByPath.find(worker.store.printStorePath(path));
-                if (i != outputsByPath.end()) {
-                    closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
-                } else {
-                    auto info = worker.store.queryPathInfo(path);
-                    closureSize += info->narSize;
-                    for (auto & ref : info->references)
-                        pathsLeft.push(ref);
-                }
-            }
-
-            return std::make_pair(std::move(pathsDone), closureSize);
-        };
-
-        auto applyChecks = [&](const Checks & checks)
-        {
-            if (checks.maxSize && info.narSize > *checks.maxSize)
-                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
-            if (checks.maxClosureSize) {
-                uint64_t closureSize = getClosure(info.path).second;
-                if (closureSize > *checks.maxClosureSize)
-                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
-            }
-
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
-            {
-                if (!value) return;
-
-                /* Parse a list of reference specifiers.  Each element must
-                   either be a store path, or the symbolic name of the output
-                   of the derivation (such as `out'). */
-                StorePathSet spec;
-                for (auto & i : *value) {
-                    if (worker.store.isStorePath(i))
-                        spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
-                        spec.insert(finalOutputs.at(i));
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
-                }
-
-                auto used = recursive
-                    ? getClosure(info.path).first
-                    : info.references;
-
-                if (recursive && checks.ignoreSelfRefs)
-                    used.erase(info.path);
-
-                StorePathSet badPaths;
-
-                for (auto & i : used)
-                    if (allowed) {
-                        if (!spec.count(i))
-                            badPaths.insert(i);
-                    } else {
-                        if (spec.count(i))
-                            badPaths.insert(i);
-                    }
-
-                if (!badPaths.empty()) {
-                    string badPathsStr;
-                    for (auto & i : badPaths) {
-                        badPathsStr += "\n  ";
-                        badPathsStr += worker.store.printStorePath(i);
-                    }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
-                        worker.store.printStorePath(info.path), badPathsStr);
-                }
-            };
-
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
-        };
-
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
-                    Checks checks;
-
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
-    }
-}
-
-
 Path DerivationGoal::openLogFile()
 {
     logSize = 0;
@@ -3722,26 +1146,15 @@ void DerivationGoal::closeLogFile()
 }
 
 
-void DerivationGoal::deleteTmpDir(bool force)
+bool DerivationGoal::isReadDesc(int fd)
 {
-    if (tmpDir != "") {
-        /* Don't keep temporary directories for builtins because they
-           might have privileged stuff (like a copy of netrc). */
-        if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError("note: keeping build directory '%s'", tmpDir);
-            chmod(tmpDir.c_str(), 0755);
-        }
-        else
-            deletePath(tmpDir);
-        tmpDir = "";
-    }
+    return fd == hook->builderOut.readSide.get();
 }
 
 
 void DerivationGoal::handleChildOutput(int fd, const string & data)
 {
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
+    if (isReadDesc(fd))
     {
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
@@ -3857,22 +1270,6 @@ void DerivationGoal::checkPathValidity()
 }
 
 
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
-    return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
-}
-
-
 void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
 {
     result.status = status;
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 6dc164922..c85bcd84f 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -2,7 +2,8 @@
 
 #include "parsed-derivations.hh"
 #include "lock.hh"
-#include "local-store.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
 #include "goal.hh"
 
 namespace nix {
@@ -79,18 +80,6 @@ struct DerivationGoal : public Goal
 
     std::map<std::string, InitialOutput> initialOutputs;
 
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
     /* File descriptor for the log file. */
     AutoCloseFD fdLogFile;
     std::shared_ptr<BufferedSink> logFileSink, logSink;
@@ -106,79 +95,15 @@ struct DerivationGoal : public Goal
 
     std::string currentHookLine;
 
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
     /* The build hook. */
     std::unique_ptr<HookInstance> hook;
 
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
     /* The sort of derivation we are building. */
     DerivationType derivationType;
 
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
     typedef void (DerivationGoal::*GoalState)();
     GoalState state;
 
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
     /* The final output paths of the build.
 
        - For input-addressed derivations, always the precomputed paths
@@ -191,11 +116,6 @@ struct DerivationGoal : public Goal
 
     BuildMode buildMode;
 
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
     BuildResult result;
 
     /* The current round, if we're building multiple times. */
@@ -203,17 +123,6 @@ struct DerivationGoal : public Goal
 
     size_t nrRounds;
 
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
     std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
 
     std::unique_ptr<Activity> act;
@@ -226,39 +135,13 @@ struct DerivationGoal : public Goal
     /* The remote machine on which we're building. */
     std::string machineName;
 
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
     DerivationGoal(const StorePath & drvPath,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
     DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
+    virtual ~DerivationGoal();
 
     void timedOut(Error && ex) override;
 
@@ -280,7 +163,7 @@ struct DerivationGoal : public Goal
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
-    void tryLocalBuild();
+    virtual void tryLocalBuild();
     void buildDone();
 
     void resolvedFinished();
@@ -288,40 +171,11 @@ struct DerivationGoal : public Goal
     /* Is the build hook willing to perform the build? */
     HookReply tryBuildHook();
 
-    /* Start building a derivation. */
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
+    virtual int getChildStatus();
 
     /* Check that the derivation outputs all exist and register them
        as valid. */
-    void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+    virtual void registerOutputs();
 
     /* Open a log file and a pipe to it. */
     Path openLogFile();
@@ -329,8 +183,18 @@ struct DerivationGoal : public Goal
     /* Close the log file. */
     void closeLogFile();
 
-    /* Delete the temporary directory, if we have one. */
-    void deleteTmpDir(bool force);
+    /* Close the read side of the logger pipe. */
+    virtual void closeReadPipes();
+
+    /* Cleanup hooks for buildDone() */
+    virtual void cleanupHookFinally();
+    virtual void cleanupPreChildKill();
+    virtual void cleanupPostChildKill();
+    virtual bool cleanupDecideWhetherDiskFull();
+    virtual void cleanupPostOutputsRegisteredModeCheck();
+    virtual void cleanupPostOutputsRegisteredModeNonCheck();
+
+    virtual bool isReadDesc(int fd);
 
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
@@ -347,17 +211,7 @@ struct DerivationGoal : public Goal
     void checkPathValidity();
 
     /* Forcibly kill the child process, if any. */
-    void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
+    virtual void killChild();
 
     void repairClosure();
 
@@ -370,4 +224,6 @@ struct DerivationGoal : public Goal
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 
+MakeError(NotDeterministic, BuildError);
+
 }
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 3a05a022c..01a564aba 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -2,6 +2,7 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "derivation-goal.hh"
+#include "local-store.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 924c69fb7..23ffe740a 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1,4 +1,4 @@
-#include "derivation-goal.hh"
+#include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 #include "worker.hh"
 #include "builtins.hh"
@@ -75,7 +75,6 @@ void handleDiffHook(
             diffHookOptions.uid = uid;
             diffHookOptions.gid = gid;
             diffHookOptions.chdir = "/";
-
             auto diffRes = runProgram(diffHookOptions);
             if (!statusOk(diffRes.first))
                 throw ExecError(diffRes.first,
@@ -94,8 +93,9 @@ void handleDiffHook(
     }
 }
 
-const Path DerivationGoal::homeDir = "/homeless-shelter";
+const Path LocalDerivationGoal::homeDir = "/homeless-shelter";
 
+#if 0
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker)
@@ -138,19 +138,20 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
        garbage-collected. (See isActiveTempFile() in gc.cc.) */
     worker.store.addTempRoot(this->drvPath);
 }
+#endif
 
 
-DerivationGoal::~DerivationGoal()
+LocalDerivationGoal::~LocalDerivationGoal()
 {
     /* Careful: we should never ever throw an exception from a
        destructor. */
+    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
     try { killChild(); } catch (...) { ignoreException(); }
     try { stopDaemon(); } catch (...) { ignoreException(); }
-    try { deleteTmpDir(false); } catch (...) { ignoreException(); }
-    try { closeLogFile(); } catch (...) { ignoreException(); }
 }
 
 
+#if 0
 string DerivationGoal::key()
 {
     /* Ensure that derivations get built in order of their name,
@@ -159,9 +160,10 @@ string DerivationGoal::key()
        derivation goals (due to "b$"). */
     return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
 }
+#endif
 
 
-inline bool DerivationGoal::needsHashRewrite()
+inline bool LocalDerivationGoal::needsHashRewrite()
 {
 #if __linux__
     return !useChroot;
@@ -172,7 +174,15 @@ inline bool DerivationGoal::needsHashRewrite()
 }
 
 
-void DerivationGoal::killChild()
+LocalStore & LocalDerivationGoal::getLocalStore()
+{
+    auto p = dynamic_cast<LocalStore *>(&worker.store);
+    assert(p);
+    return *p;
+}
+
+
+void LocalDerivationGoal::killChild()
 {
     if (pid != -1) {
         worker.childTerminated(this);
@@ -193,17 +203,11 @@ void DerivationGoal::killChild()
         assert(pid == -1);
     }
 
-    hook.reset();
-}
-
-
-void DerivationGoal::timedOut(Error && ex)
-{
-    killChild();
-    done(BuildResult::TimedOut, ex);
+    DerivationGoal::killChild();
 }
 
 
+#if 0
 void DerivationGoal::work()
 {
     (this->*state)();
@@ -695,15 +699,9 @@ void DerivationGoal::tryToBuild()
     state = &DerivationGoal::tryLocalBuild;
     worker.wakeUp(shared_from_this());
 }
+#endif
 
-void DerivationGoal::tryLocalBuild() {
-    /* Make sure that we are allowed to start a build. */
-    if (!dynamic_cast<LocalStore *>(&worker.store)) {
-        throw Error(
-            "unable to build with a primary store that isn't a local store; "
-            "either pass a different '--store' or enable remote builds."
-            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
-    }
+void LocalDerivationGoal::tryLocalBuild() {
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
         worker.waitForBuildSlot(shared_from_this());
@@ -757,7 +755,6 @@ void DerivationGoal::tryLocalBuild() {
     started();
 }
 
-
 static void chmod_(const Path & path, mode_t mode)
 {
     if (chmod(path.c_str(), mode) == -1)
@@ -785,51 +782,125 @@ static void movePath(const Path & src, const Path & dst)
 }
 
 
-void replaceValidPath(const Path & storePath, const Path & tmpPath)
+extern void replaceValidPath(const Path & storePath, const Path & tmpPath);
+
+
+int LocalDerivationGoal::getChildStatus()
 {
-    /* We can't atomically replace storePath (the original) with
-       tmpPath (the replacement), so we have to move it out of the
-       way first.  We'd better not be interrupted here, because if
-       we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
-    if (pathExists(storePath))
-        movePath(storePath, oldPath);
+    return hook ? DerivationGoal::getChildStatus() : pid.kill();
+}
 
-    try {
-        movePath(tmpPath, storePath);
-    } catch (...) {
-        try {
-            // attempt to recover
-            movePath(oldPath, storePath);
-        } catch (...) {
-            ignoreException();
-        }
-        throw;
-    }
-
-    deletePath(oldPath);
+void LocalDerivationGoal::closeReadPipes()
+{
+    if (hook) {
+        DerivationGoal::closeReadPipes();
+    } else
+        builderOut.readSide = -1;
 }
 
 
-MakeError(NotDeterministic, BuildError);
+void LocalDerivationGoal::cleanupHookFinally()
+{
+    /* Release the build user at the end of this function. We don't do
+       it right away because we don't want another build grabbing this
+       uid and then messing around with our output. */
+    buildUser.reset();
+}
 
 
+void LocalDerivationGoal::cleanupPreChildKill()
+{
+    sandboxMountNamespace = -1;
+}
+
+
+void LocalDerivationGoal::cleanupPostChildKill()
+{
+    /* When running under a build user, make sure that all processes
+       running under that uid are gone.  This is to prevent a
+       malicious user from leaving behind a process that keeps files
+       open and modifies them after they have been chown'ed to
+       root. */
+    if (buildUser) buildUser->kill();
+
+    /* Terminate the recursive Nix daemon. */
+    stopDaemon();
+}
+
+
+bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
+{
+    bool diskFull = false;
+
+    /* Heuristically check whether the build failure may have
+       been caused by a disk full condition.  We have no way
+       of knowing whether the build actually got an ENOSPC.
+       So instead, check if the disk is (nearly) full now.  If
+       so, we don't mark this build as a permanent failure. */
+#if HAVE_STATVFS
+	{
+        auto & localStore = getLocalStore();
+        uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
+        struct statvfs st;
+        if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
+            (uint64_t) st.f_bavail * st.f_bsize < required)
+            diskFull = true;
+        if (statvfs(tmpDir.c_str(), &st) == 0 &&
+            (uint64_t) st.f_bavail * st.f_bsize < required)
+            diskFull = true;
+	}
+#endif
+
+    deleteTmpDir(false);
+
+    /* Move paths out of the chroot for easier debugging of
+       build failures. */
+    if (useChroot && buildMode == bmNormal)
+        for (auto & [_, status] : initialOutputs) {
+            if (!status.known) continue;
+            if (buildMode != bmCheck && status.known->isValid()) continue;
+            auto p = worker.store.printStorePath(status.known->path);
+            if (pathExists(chrootRootDir + p))
+                rename((chrootRootDir + p).c_str(), p.c_str());
+        }
+
+    return diskFull;
+}
+
+
+void LocalDerivationGoal::cleanupPostOutputsRegisteredModeCheck()
+{
+    deleteTmpDir(true);
+}
+
+
+void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
+{
+    /* Delete unused redirected outputs (when doing hash rewriting). */
+    for (auto & i : redirectedOutputs)
+        deletePath(worker.store.Store::toRealPath(i.second));
+
+    /* Delete the chroot (if we were using one). */
+    autoDelChroot.reset(); /* this runs the destructor */
+
+    cleanupPostOutputsRegisteredModeCheck();
+}
+
+
+#if 0
 void DerivationGoal::buildDone()
 {
     trace("build done");
 
-    /* Release the build user at the end of this function. We don't do
-       it right away because we don't want another build grabbing this
-       uid and then messing around with our output. */
-    Finally releaseBuildUser([&]() { buildUser.reset(); });
+    Finally releaseBuildUser([&](){ this->cleanupHookFinally(); });
 
-    sandboxMountNamespace = -1;
+    cleanupPreChildKill();
 
     /* Since we got an EOF on the logger pipe, the builder is presumed
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe, so just to be sure,
        kill it. */
-    int status = hook ? hook->pid.kill() : pid.kill();
+    int status = getChildStatus();
 
     debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
 
@@ -840,24 +911,12 @@ void DerivationGoal::buildDone()
     worker.childTerminated(this);
 
     /* Close the read side of the logger pipe. */
-    if (hook) {
-        hook->builderOut.readSide = -1;
-        hook->fromHook.readSide = -1;
-    } else
-        builderOut.readSide = -1;
+    closeReadPipes();
 
     /* Close the log file. */
     closeLogFile();
 
-    /* When running under a build user, make sure that all processes
-       running under that uid are gone.  This is to prevent a
-       malicious user from leaving behind a process that keeps files
-       open and modifies them after they have been chown'ed to
-       root. */
-    if (buildUser) buildUser->kill();
-
-    /* Terminate the recursive Nix daemon. */
-    stopDaemon();
+    cleanupPostChildKill();
 
     bool diskFull = false;
 
@@ -866,36 +925,7 @@ void DerivationGoal::buildDone()
         /* Check the exit status. */
         if (!statusOk(status)) {
 
-            /* Heuristically check whether the build failure may have
-               been caused by a disk full condition.  We have no way
-               of knowing whether the build actually got an ENOSPC.
-               So instead, check if the disk is (nearly) full now.  If
-               so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
-            if (auto localStore = dynamic_cast<LocalStore *>(&worker.store)) {
-                uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
-                struct statvfs st;
-                if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 &&
-                    (uint64_t) st.f_bavail * st.f_bsize < required)
-                    diskFull = true;
-                if (statvfs(tmpDir.c_str(), &st) == 0 &&
-                    (uint64_t) st.f_bavail * st.f_bsize < required)
-                    diskFull = true;
-            }
-#endif
-
-            deleteTmpDir(false);
-
-            /* Move paths out of the chroot for easier debugging of
-               build failures. */
-            if (useChroot && buildMode == bmNormal)
-                for (auto & [_, status] : initialOutputs) {
-                    if (!status.known) continue;
-                    if (buildMode != bmCheck && status.known->isValid()) continue;
-                    auto p = worker.store.printStorePath(status.known->path);
-                    if (pathExists(chrootRootDir + p))
-                        rename((chrootRootDir + p).c_str(), p.c_str());
-                }
+            diskFull |= cleanupDecideWhetherDiskFull();
 
             auto msg = fmt("builder for '%s' %s",
                 yellowtxt(worker.store.printStorePath(drvPath)),
@@ -975,19 +1005,12 @@ void DerivationGoal::buildDone()
         }
 
         if (buildMode == bmCheck) {
-            deleteTmpDir(true);
+            cleanupPostOutputsRegisteredModeCheck();
             done(BuildResult::Built);
             return;
         }
 
-        /* Delete unused redirected outputs (when doing hash rewriting). */
-        for (auto & i : redirectedOutputs)
-            deletePath(worker.store.Store::toRealPath(i.second));
-
-        /* Delete the chroot (if we were using one). */
-        autoDelChroot.reset(); /* this runs the destructor */
-
-        deleteTmpDir(true);
+        cleanupPostOutputsRegisteredModeNonCheck();
 
         /* Repeat the build if necessary. */
         if (curRound++ < nrRounds) {
@@ -1169,15 +1192,17 @@ HookReply DerivationGoal::tryBuildHook()
 
     return rpAccept;
 }
+#endif
 
 
 int childEntry(void * arg)
 {
-    ((DerivationGoal *) arg)->runChild();
+    ((LocalDerivationGoal *) arg)->runChild();
     return 1;
 }
 
 
+#if 0
 StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
 {
     StorePathSet paths;
@@ -1212,6 +1237,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
 
     return paths;
 }
+#endif
 
 static std::once_flag dns_resolve_flag;
 
@@ -1230,7 +1256,7 @@ static void preloadNSS() {
 }
 
 
-void linkOrCopy(const Path & from, const Path & to)
+static void linkOrCopy(const Path & from, const Path & to)
 {
     if (link(from.c_str(), to.c_str()) == -1) {
         /* Hard-linking fails if we exceed the maximum link count on a
@@ -1247,7 +1273,7 @@ void linkOrCopy(const Path & from, const Path & to)
 }
 
 
-void DerivationGoal::startBuilder()
+void LocalDerivationGoal::startBuilder()
 {
     /* Right platform? */
     if (!parsedDrv->canBuildLocally(worker.store))
@@ -1285,15 +1311,13 @@ void DerivationGoal::startBuilder()
             useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
     }
 
-    if (auto localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
-        auto & localStore = *localStoreP;
-        if (localStore.storeDir != localStore.realStoreDir) {
-            #if __linux__
-                useChroot = true;
-            #else
-                throw Error("building using a diverted store is not supported on this platform");
-            #endif
-        }
+    auto & localStore = getLocalStore();
+    if (localStore.storeDir != localStore.realStoreDir) {
+        #if __linux__
+            useChroot = true;
+        #else
+            throw Error("building using a diverted store is not supported on this platform");
+        #endif
     }
 
     /* Create a temporary directory where the build will take
@@ -1850,7 +1874,7 @@ void DerivationGoal::startBuilder()
 }
 
 
-void DerivationGoal::initTmpDir() {
+void LocalDerivationGoal::initTmpDir() {
     /* In a sandbox, for determinism, always use the same temporary
        directory. */
 #if __linux__
@@ -1899,7 +1923,7 @@ void DerivationGoal::initTmpDir() {
 }
 
 
-void DerivationGoal::initEnv()
+void LocalDerivationGoal::initEnv()
 {
     env.clear();
 
@@ -1960,7 +1984,7 @@ void DerivationGoal::initEnv()
 static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 
-void DerivationGoal::writeStructuredAttrs()
+void LocalDerivationGoal::writeStructuredAttrs()
 {
     auto structuredAttrs = parsedDrv->getStructuredAttrs();
     if (!structuredAttrs) return;
@@ -2079,9 +2103,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 {
     ref<LocalStore> next;
 
-    DerivationGoal & goal;
+    LocalDerivationGoal & goal;
 
-    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
+    RestrictedStore(const Params & params, ref<LocalStore> next, LocalDerivationGoal & goal)
         : StoreConfig(params)
         , LocalFSStoreConfig(params)
         , RestrictedStoreConfig(params)
@@ -2256,15 +2280,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 };
 
 
-void DerivationGoal::startDaemon()
+void LocalDerivationGoal::startDaemon()
 {
     settings.requireExperimentalFeature("recursive-nix");
 
     Store::Params params;
     params["path-info-cache-size"] = "0";
     params["store"] = worker.store.storeDir;
-    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
-        params["root"] = localStore->rootDir;
+    params["root"] = getLocalStore().rootDir;
     params["state"] = "/no-such-path";
     params["log"] = "/no-such-path";
     auto store = make_ref<RestrictedStore>(params,
@@ -2322,7 +2345,7 @@ void DerivationGoal::startDaemon()
 }
 
 
-void DerivationGoal::stopDaemon()
+void LocalDerivationGoal::stopDaemon()
 {
     if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
         throw SysError("shutting down daemon socket");
@@ -2340,7 +2363,7 @@ void DerivationGoal::stopDaemon()
 }
 
 
-void DerivationGoal::addDependency(const StorePath & path)
+void LocalDerivationGoal::addDependency(const StorePath & path)
 {
     if (isAllowed(path)) return;
 
@@ -2397,8 +2420,7 @@ void DerivationGoal::addDependency(const StorePath & path)
     }
 }
 
-
-void DerivationGoal::chownToBuilder(const Path & path)
+void LocalDerivationGoal::chownToBuilder(const Path & path)
 {
     if (!buildUser) return;
     if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
@@ -2469,7 +2491,7 @@ void setupSeccomp()
 }
 
 
-void DerivationGoal::runChild()
+void LocalDerivationGoal::runChild()
 {
     /* Warning: in the child we should absolutely not make any SQLite
        calls! */
@@ -2977,7 +2999,7 @@ void DerivationGoal::runChild()
 }
 
 
-void DerivationGoal::registerOutputs()
+void LocalDerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
        as valid (by doing `nix-store --import').  If so we don't have
@@ -2987,14 +3009,8 @@ void DerivationGoal::registerOutputs()
        floating content-addressed derivations this isn't the case.
      */
     if (hook) {
-        bool allValid = true;
-        for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
-            if (!outputPath || !worker.store.isValidPath(*outputPath))
-                allValid = false;
-            else
-                finalOutputs.insert_or_assign(outputName, *outputPath);
-        }
-        if (allValid) return;
+        DerivationGoal::registerOutputs();
+        return;
     }
 
     std::map<std::string, ValidPathInfo> infos;
@@ -3349,10 +3365,7 @@ void DerivationGoal::registerOutputs()
             }
         }
 
-        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
-        if (!localStoreP)
-            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
-        auto & localStore = *localStoreP;
+        auto & localStore = getLocalStore();
 
         if (buildMode == bmCheck) {
 
@@ -3481,10 +3494,7 @@ void DerivationGoal::registerOutputs()
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
     {
-        auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
-        if (!localStoreP)
-            throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
-        auto & localStore = *localStoreP;
+        auto & localStore = getLocalStore();
 
         ValidPathInfos infos2;
         for (auto & [outputName, newInfo] : infos) {
@@ -3513,7 +3523,7 @@ void DerivationGoal::registerOutputs()
 }
 
 
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
+void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 {
     std::map<Path, const ValidPathInfo &> outputsByPath;
     for (auto & output : outputs)
@@ -3678,6 +3688,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 }
 
 
+#if 0
 Path DerivationGoal::openLogFile()
 {
     logSize = 0;
@@ -3720,9 +3731,10 @@ void DerivationGoal::closeLogFile()
     logSink = logFileSink = 0;
     fdLogFile = -1;
 }
+#endif
 
 
-void DerivationGoal::deleteTmpDir(bool force)
+void LocalDerivationGoal::deleteTmpDir(bool force)
 {
     if (tmpDir != "") {
         /* Don't keep temporary directories for builtins because they
@@ -3738,10 +3750,17 @@ void DerivationGoal::deleteTmpDir(bool force)
 }
 
 
+bool LocalDerivationGoal::isReadDesc(int fd)
+{
+    return (hook && DerivationGoal::isReadDesc(fd)) ||
+        (!hook && fd == builderOut.readSide.get());
+}
+
+
+#if 0
 void DerivationGoal::handleChildOutput(int fd, const string & data)
 {
-    if ((hook && fd == hook->builderOut.readSide.get()) ||
-        (!hook && fd == builderOut.readSide.get()))
+    if (isReadDesc(fd))
     {
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
@@ -3855,9 +3874,10 @@ void DerivationGoal::checkPathValidity()
         }
     }
 }
+#endif
 
 
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
+StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
@@ -3865,7 +3885,7 @@ StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
 }
 
 
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
+StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
@@ -3873,6 +3893,7 @@ StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
 }
 
 
+#if 0
 void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
 {
     result.status = status;
@@ -3897,6 +3918,7 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
 
     worker.updateProgress();
 }
+#endif
 
 
 }
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 6dc164922..f7994113e 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -1,48 +1,15 @@
 #pragma once
 
-#include "parsed-derivations.hh"
-#include "lock.hh"
+#include "derivation-goal.hh"
 #include "local-store.hh"
-#include "goal.hh"
 
 namespace nix {
 
-using std::map;
-
-struct HookInstance;
-
-typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
-
-/* Unless we are repairing, we don't both to test validity and just assume it,
-   so the choices are `Absent` or `Valid`. */
-enum struct PathStatus {
-    Corrupt,
-    Absent,
-    Valid,
-};
-
-struct InitialOutputStatus {
-    StorePath path;
-    PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
-    bool isValid() const {
-        return status == PathStatus::Valid;
-    }
-    /* Merely present, allowed to be corrupt */
-    bool isPresent() const {
-        return status == PathStatus::Corrupt
-            || status == PathStatus::Valid;
-    }
-};
-
-struct InitialOutput {
-    bool wanted;
-    Hash outputHash;
-    std::optional<InitialOutputStatus> known;
-};
-
-struct DerivationGoal : public Goal
+struct LocalDerivationGoal : public DerivationGoal
 {
+    LocalStore & getLocalStore();
+
+#if 0
     /* Whether to use an on-disk .drv file. */
     bool useDerivation;
 
@@ -78,6 +45,7 @@ struct DerivationGoal : public Goal
     StorePathSet inputPaths;
 
     std::map<std::string, InitialOutput> initialOutputs;
+#endif
 
     /* User selected for running the builder. */
     std::unique_ptr<UserLock> buildUser;
@@ -91,6 +59,7 @@ struct DerivationGoal : public Goal
     /* The path of the temporary directory in the sandbox. */
     Path tmpDirInSandbox;
 
+#if 0
     /* File descriptor for the log file. */
     AutoCloseFD fdLogFile;
     std::shared_ptr<BufferedSink> logFileSink, logSink;
@@ -105,6 +74,7 @@ struct DerivationGoal : public Goal
     size_t currentLogLinePos = 0; // to handle carriage return
 
     std::string currentHookLine;
+#endif
 
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
@@ -120,8 +90,10 @@ struct DerivationGoal : public Goal
        namespace. */
     bool usingUserNamespace = true;
 
+#if 0
     /* The build hook. */
     std::unique_ptr<HookInstance> hook;
+#endif
 
     /* Whether we're currently doing a chroot build. */
     bool useChroot = false;
@@ -131,14 +103,18 @@ struct DerivationGoal : public Goal
     /* RAII object to delete the chroot directory. */
     std::shared_ptr<AutoDelete> autoDelChroot;
 
+#if 0
     /* The sort of derivation we are building. */
     DerivationType derivationType;
+#endif
 
     /* Whether to run the build in a private network namespace. */
     bool privateNetwork = false;
 
+#if 0
     typedef void (DerivationGoal::*GoalState)();
     GoalState state;
+#endif
 
     /* Stuff we need to pass to initChild(). */
     struct ChrootPath {
@@ -179,6 +155,7 @@ struct DerivationGoal : public Goal
      */
     OutputPathMap scratchOutputs;
 
+#if 0
     /* The final output paths of the build.
 
        - For input-addressed derivations, always the precomputed paths
@@ -190,18 +167,21 @@ struct DerivationGoal : public Goal
     OutputPathMap finalOutputs;
 
     BuildMode buildMode;
+#endif
 
     /* If we're repairing without a chroot, there may be outputs that
        are valid but corrupt.  So we redirect these outputs to
        temporary paths. */
     StorePathSet redirectedBadOutputs;
 
+#if 0
     BuildResult result;
 
     /* The current round, if we're building multiple times. */
     size_t curRound = 1;
 
     size_t nrRounds;
+#endif
 
     /* Path registration info from the previous round, if we're
        building multiple times. Since this contains the hash, it
@@ -214,6 +194,7 @@ struct DerivationGoal : public Goal
 
     const static Path homeDir;
 
+#if 0
     std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
 
     std::unique_ptr<Activity> act;
@@ -225,6 +206,7 @@ struct DerivationGoal : public Goal
 
     /* The remote machine on which we're building. */
     std::string machineName;
+#endif
 
     /* The recursive Nix daemon socket. */
     AutoCloseFD daemonSocket;
@@ -249,17 +231,14 @@ struct DerivationGoal : public Goal
 
     friend struct RestrictedStore;
 
-    DerivationGoal(const StorePath & drvPath,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-        const StringSet & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
+    using DerivationGoal::DerivationGoal;
+
+    virtual ~LocalDerivationGoal() override;
 
     /* Whether we need to perform hash rewriting if there are valid output paths. */
     bool needsHashRewrite();
 
+#if 0
     void timedOut(Error && ex) override;
 
     string key() override;
@@ -280,13 +259,16 @@ struct DerivationGoal : public Goal
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
-    void tryLocalBuild();
+#endif
+    void tryLocalBuild() override;
+#if 0
     void buildDone();
 
     void resolvedFinished();
 
     /* Is the build hook willing to perform the build? */
     HookReply tryBuildHook();
+#endif
 
     /* Start building a derivation. */
     void startBuilder();
@@ -311,27 +293,46 @@ struct DerivationGoal : public Goal
     /* Make a file owned by the builder. */
     void chownToBuilder(const Path & path);
 
+    int getChildStatus() override;
+
     /* Run the builder's process. */
     void runChild();
 
     /* Check that the derivation outputs all exist and register them
        as valid. */
-    void registerOutputs();
+    void registerOutputs() override;
 
     /* Check that an output meets the requirements specified by the
        'outputChecks' attribute (or the legacy
        '{allowed,disallowed}{References,Requisites}' attributes). */
     void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
 
+#if 0
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
     /* Close the log file. */
     void closeLogFile();
+#endif
+
+    /* Close the read side of the logger pipe. */
+    void closeReadPipes() override;
+
+    /* Cleanup hooks for buildDone() */
+    void cleanupHookFinally() override;
+    void cleanupPreChildKill() override;
+    void cleanupPostChildKill() override;
+    bool cleanupDecideWhetherDiskFull() override;
+    void cleanupPostOutputsRegisteredModeCheck() override;
+    void cleanupPostOutputsRegisteredModeNonCheck() override;
+
+    bool isReadDesc(int fd) override;
+
 
     /* Delete the temporary directory, if we have one. */
     void deleteTmpDir(bool force);
 
+#if 0
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
     void handleEOF(int fd) override;
@@ -345,9 +346,10 @@ struct DerivationGoal : public Goal
 
     /* Return the set of (in)valid paths. */
     void checkPathValidity();
+#endif
 
     /* Forcibly kill the child process, if any. */
-    void killChild();
+    void killChild() override;
 
     /* Create alternative path calculated from but distinct from the
        input, so we can avoid overwriting outputs (or other store paths)
@@ -359,6 +361,7 @@ struct DerivationGoal : public Goal
        rewrites caught everything */
     StorePath makeFallbackPath(std::string_view outputName);
 
+#if 0
     void repairClosure();
 
     void started();
@@ -368,6 +371,7 @@ struct DerivationGoal : public Goal
         std::optional<Error> ex = {});
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
+#endif
 };
 
 }
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 2f13aa885..b2223c3b6 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -1,7 +1,7 @@
 #include "machines.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
-#include "derivation-goal.hh"
+#include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
 #include <poll.h>
@@ -59,8 +59,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
     const StringSet & wantedOutputs, BuildMode buildMode)
 {
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
+        return !dynamic_cast<LocalStore *>(&store)
+            ? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
+            : std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
     });
 }
 
@@ -68,8 +70,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
     const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
 {
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
+        return !dynamic_cast<LocalStore *>(&store)
+            ? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
+            : std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
     });
 }
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 780cc0f07..03bb0218d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -280,7 +280,7 @@ private:
 
     void createUser(const std::string & userName, uid_t userId) override;
 
-    friend struct DerivationGoal;
+    friend struct LocalDerivationGoal;
     friend struct SubstitutionGoal;
 };
 

From d560311f7643096ce815a7c655a077621abb7d1a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 Feb 2021 15:31:15 +0000
Subject: [PATCH 777/882] Remove temporary `#if 0...#endif` from previous
 commit

---
 src/libstore/build/local-derivation-goal.cc | 1090 -------------------
 src/libstore/build/local-derivation-goal.hh |  175 +--
 2 files changed, 1 insertion(+), 1264 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 23ffe740a..3a0616864 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -95,51 +95,6 @@ void handleDiffHook(
 
 const Path LocalDerivationGoal::homeDir = "/homeless-shelter";
 
-#if 0
-DerivationGoal::DerivationGoal(const StorePath & drvPath,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(true)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &DerivationGoal::getDerivation;
-    name = fmt(
-        "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-}
-
-
-DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
-    const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
-    , useDerivation(false)
-    , drvPath(drvPath)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    this->drv = std::make_unique<Derivation>(drv);
-
-    state = &DerivationGoal::haveDerivation;
-    name = fmt(
-        "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
-    trace("created");
-
-    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
-    worker.updateProgress();
-
-    /* Prevent the .chroot directory from being
-       garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(this->drvPath);
-}
-#endif
-
 
 LocalDerivationGoal::~LocalDerivationGoal()
 {
@@ -151,18 +106,6 @@ LocalDerivationGoal::~LocalDerivationGoal()
 }
 
 
-#if 0
-string DerivationGoal::key()
-{
-    /* Ensure that derivations get built in order of their name,
-       i.e. a derivation named "aardvark" always comes before
-       "baboon". And substitution goals always happen before
-       derivation goals (due to "b$"). */
-    return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
-}
-#endif
-
-
 inline bool LocalDerivationGoal::needsHashRewrite()
 {
 #if __linux__
@@ -207,500 +150,6 @@ void LocalDerivationGoal::killChild()
 }
 
 
-#if 0
-void DerivationGoal::work()
-{
-    (this->*state)();
-}
-
-
-void DerivationGoal::addWantedOutputs(const StringSet & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    if (wantedOutputs.empty()) return;
-
-    if (outputs.empty()) {
-        wantedOutputs.clear();
-        needRestart = true;
-    } else
-        for (auto & i : outputs)
-            if (wantedOutputs.insert(i).second)
-                needRestart = true;
-}
-
-
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath)));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
-void DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
-
-    assert(worker.store.isValidPath(drvPath));
-
-    /* Get the derivation. */
-    drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
-
-    haveDerivation();
-}
-
-
-void DerivationGoal::haveDerivation()
-{
-    trace("have derivation");
-
-    if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
-
-    retrySubstitution = false;
-
-    for (auto & i : drv->outputsAndOptPaths(worker.store))
-        if (i.second.second)
-            worker.store.addTempRoot(*i.second.second);
-
-    auto outputHashes = staticOutputHashes(worker.store, *drv);
-    for (auto &[outputName, outputHash] : outputHashes)
-      initialOutputs.insert({
-            outputName,
-            InitialOutput{
-                .wanted = true, // Will be refined later
-                .outputHash = outputHash
-            }
-          });
-
-    /* Check what outputs paths are not already valid. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
-    /* We are first going to try to create the invalid output paths
-       through substitutes.  If that doesn't work, we'll build
-       them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
-            if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(upcast_goal(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv))));
-        }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
-}
-
-
-void DerivationGoal::outputsSubstitutionTried()
-{
-    trace("all outputs substituted (maybe)");
-
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
-                worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    /*  If the substitutes form an incomplete closure, then we should
-        build the dependencies of this derivation, but after that, we
-        can still use the substitutes for this derivation itself.
-
-        If the nrIncompleteClosure != nrFailed, we have another issue as well.
-        In particular, it may be the case that the hole in the closure is
-        an output of the current derivation, which causes a loop if retried.
-     */
-    if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true;
-
-    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-
-    if (needRestart) {
-        needRestart = false;
-        haveDerivation();
-        return;
-    }
-
-    checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
-
-    if (buildMode == bmNormal && nrInvalid == 0) {
-        done(BuildResult::Substituted);
-        return;
-    }
-    if (buildMode == bmRepair && nrInvalid == 0) {
-        repairClosure();
-        return;
-    }
-    if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error("some outputs of '%s' are not valid, so checking is not possible",
-            worker.store.printStorePath(drvPath));
-
-    /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
-}
-
-/* At least one of the output paths could not be
-   produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
-{
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
-    /* The inputs must be built before we can build this goal. */
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
-
-    for (auto & i : drv->inputSrcs) {
-        if (worker.store.isValidPath(i)) continue;
-        if (!settings.useSubstitutes)
-            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
-                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(upcast_goal(worker.makeSubstitutionGoal(i)));
-    }
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
-}
-
-
-void DerivationGoal::repairClosure()
-{
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantOutput(i.first, wantedOutputs)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        printError(
-            "found corrupted or missing path '%s' in the output closure of '%s'",
-            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
-        else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
-    }
-
-    if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    state = &DerivationGoal::closureRepaired;
-}
-
-
-void DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
-}
-
-
-void DerivationGoal::inputsRealised()
-{
-    trace("all inputs realised");
-
-    if (nrFailed != 0) {
-        if (!useDerivation)
-            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
-                "%s dependencies of derivation '%s' failed to build",
-                nrFailed, worker.store.printStorePath(drvPath)));
-        return;
-    }
-
-    if (retrySubstitution) {
-        haveDerivation();
-        return;
-    }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-
-    /* Determine the full set of input paths. */
-
-    /* First, the input derivations. */
-    if (useDerivation) {
-        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
-
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
-            ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
-            || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
-            /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
-               aliasing that resolved derivation goal */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
-            assert(attempt);
-            Derivation drvResolved { *std::move(attempt) };
-
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
-            resolvedDrv = drvResolved;
-
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
-                worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
-            act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
-                Logger::Fields {
-                       worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
-                   });
-
-            auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
-
-            state = &DerivationGoal::resolvedFinished;
-            return;
-        }
-
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
-            /* Add the relevant output closures of the input derivation
-               `i' as input paths.  Only add the closures of output paths
-               that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
-            for (auto & j : wantedDepOutputs) {
-                if (outputs.count(j) > 0) {
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
-            }
-        }
-    }
-
-    /* Second, the input sources. */
-    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
-
-    debug("added input paths %s", worker.store.showPaths(inputPaths));
-
-    /* What type of derivation are we building? */
-    derivationType = drv->type();
-
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
-
-    /* Okay, try to build.  Note that here we don't wait for a build
-       slot to become available, since we don't need one if there is a
-       build hook. */
-    state = &DerivationGoal::tryToBuild;
-    worker.wakeUp(shared_from_this());
-
-    result = BuildResult();
-}
-
-void DerivationGoal::started() {
-    auto msg = fmt(
-        buildMode == bmRepair ? "repairing outputs of '%s'" :
-        buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-    if (hook) msg += fmt(" on '%s'", machineName);
-    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
-void DerivationGoal::tryToBuild()
-{
-    trace("trying to build");
-
-    /* Obtain locks on all output paths, if the paths are known a priori.
-
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    if (dynamic_cast<LocalStore *>(&worker.store))
-        /* If we aren't a local store, we might need to use the local store as
-           a build remote, but that would cause a deadlock. */
-        /* FIXME: Make it so we can use ourselves as a build remote even if we
-           are the local store (separate locking for building vs scheduling? */
-        /* FIXME: find some way to lock for scheduling for the other stores so
-           a forking daemon with --store still won't farm out redundant builds.
-           */
-        for (auto & i : drv->outputsAndOptPaths(worker.store))
-            if (i.second.second)
-                lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        if (!actLock)
-            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    actLock.reset();
-
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
-        return;
-    }
-
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid()) continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
-
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-            case rpAccept:
-                /* Yes, it has started doing so.  Wait until we get
-                   EOF from the hook. */
-                actLock.reset();
-                result.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
-                started();
-                return;
-            case rpPostpone:
-                /* Not now; wait until at least one child finishes or
-                   the wake-up timeout expires. */
-                if (!actLock)
-                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
-                worker.waitForAWhile(shared_from_this());
-                outputLocks.unlock();
-                return;
-            case rpDecline:
-                /* We should do it ourselves. */
-                break;
-        }
-    }
-
-    actLock.reset();
-
-    state = &DerivationGoal::tryLocalBuild;
-    worker.wakeUp(shared_from_this());
-}
-#endif
-
 void LocalDerivationGoal::tryLocalBuild() {
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
@@ -887,314 +336,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 }
 
 
-#if 0
-void DerivationGoal::buildDone()
-{
-    trace("build done");
-
-    Finally releaseBuildUser([&](){ this->cleanupHookFinally(); });
-
-    cleanupPreChildKill();
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status = getChildStatus();
-
-    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
-
-    result.timesBuilt++;
-    result.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-    closeReadPipes();
-
-    /* Close the log file. */
-    closeLogFile();
-
-    cleanupPostChildKill();
-
-    bool diskFull = false;
-
-    try {
-
-        /* Check the exit status. */
-        if (!statusOk(status)) {
-
-            diskFull |= cleanupDecideWhetherDiskFull();
-
-            auto msg = fmt("builder for '%s' %s",
-                yellowtxt(worker.store.printStorePath(drvPath)),
-                statusToString(status));
-
-            if (!logger->isVerbose() && !logTail.empty()) {
-                msg += fmt(";\nlast %d log lines:\n", logTail.size());
-                for (auto & line : logTail) {
-                    msg += "> ";
-                    msg += line;
-                    msg += "\n";
-                }
-                msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.",
-                    worker.store.printStorePath(drvPath));
-            }
-
-            if (diskFull)
-                msg += "\nnote: build failure may have been caused by lack of free disk space";
-
-            throw BuildError(msg);
-        }
-
-        /* Compute the FS closure of the outputs and register them as
-           being valid. */
-        registerOutputs();
-
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (std::string_view data) override {
-                    for (auto c : data) {
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
-
-        if (buildMode == bmCheck) {
-            cleanupPostOutputsRegisteredModeCheck();
-            done(BuildResult::Built);
-            return;
-        }
-
-        cleanupPostOutputsRegisteredModeNonCheck();
-
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
-        /* It is now safe to delete the lock files, since all future
-           lockers will see that the output paths are valid; they will
-           not create new lock files with the same names as the old
-           (unlinked) lock files. */
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-
-    } catch (BuildError & e) {
-        outputLocks.unlock();
-
-        BuildResult::Status st = BuildResult::MiscFailure;
-
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
-            st = BuildResult::TimedOut;
-
-        else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-        }
-
-        else {
-            st =
-                dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
-                statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
-                BuildResult::PermanentFailure;
-        }
-
-        done(st, e);
-        return;
-    }
-
-    done(BuildResult::Built);
-}
-
-void DerivationGoal::resolvedFinished() {
-    assert(resolvedDrv);
-
-    auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
-
-    // `wantedOutputs` might be empty, which means “all the outputs”
-    auto realWantedOutputs = wantedOutputs;
-    if (realWantedOutputs.empty())
-        realWantedOutputs = resolvedDrv->outputNames();
-
-    for (auto & wantedOutput : realWantedOutputs) {
-        assert(initialOutputs.count(wantedOutput) != 0);
-        assert(resolvedHashes.count(wantedOutput) != 0);
-        auto realisation = worker.store.queryRealisation(
-                DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
-        );
-        // We've just built it, but maybe the build failed, in which case the
-        // realisation won't be there
-        if (realisation) {
-            auto newRealisation = *realisation;
-            newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
-            worker.store.registerDrvOutput(newRealisation);
-        } else {
-            // If we don't have a realisation, then it must mean that something
-            // failed when building the resolved drv
-            assert(!result.success());
-        }
-    }
-
-    // This is potentially a bit fishy in terms of error reporting. Not sure
-    // how to do it in a cleaner way
-    amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
-}
-
-HookReply DerivationGoal::tryBuildHook()
-{
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
-
-    if (!worker.hook)
-        worker.hook = std::make_unique<HookInstance>();
-
-    try {
-
-        /* Send the request to the hook. */
-        worker.hook->sink
-            << "try"
-            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
-            << drv->platform
-            << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
-        worker.hook->sink.flush();
-
-        /* Read the first line of input, which should be a word indicating
-           whether the hook wishes to perform the build. */
-        string reply;
-        while (true) {
-            auto s = [&]() {
-                try {
-                    return readLine(worker.hook->fromHook.readSide.get());
-                } catch (Error & e) {
-                    e.addTrace({}, "while reading the response from the build hook");
-                    throw e;
-                }
-            }();
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
-                ;
-            else if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            else {
-                s += "\n";
-                writeToStderr(s);
-            }
-        }
-
-        debug("hook reply is '%1%'", reply);
-
-        if (reply == "decline")
-            return rpDecline;
-        else if (reply == "decline-permanently") {
-            worker.tryBuildHook = false;
-            worker.hook = 0;
-            return rpDecline;
-        }
-        else if (reply == "postpone")
-            return rpPostpone;
-        else if (reply != "accept")
-            throw Error("bad hook reply '%s'", reply);
-
-    } catch (SysError & e) {
-        if (e.errNo == EPIPE) {
-            printError(
-                "build hook died unexpectedly: %s",
-                chomp(drainFD(worker.hook->fromHook.readSide.get())));
-            worker.hook = 0;
-            return rpDecline;
-        } else
-            throw;
-    }
-
-    hook = std::move(worker.hook);
-
-    try {
-        machineName = readLine(hook->fromHook.readSide.get());
-    } catch (Error & e) {
-        e.addTrace({}, "while reading the machine name from the build hook");
-        throw e;
-    }
-
-    /* Tell the hook all the inputs that have to be copied to the
-       remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
-
-    /* Tell the hooks the missing outputs that have to be copied back
-       from the remote system. */
-    {
-        StringSet missingOutputs;
-        for (auto & [outputName, status] : initialOutputs) {
-            // XXX: Does this include known CA outputs?
-            if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
-            missingOutputs.insert(outputName);
-        }
-        worker_proto::write(worker.store, hook->sink, missingOutputs);
-    }
-
-    hook->sink = FdSink();
-    hook->toHook.writeSide = -1;
-
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
-
-    set<int> fds;
-    fds.insert(hook->fromHook.readSide.get());
-    fds.insert(hook->builderOut.readSide.get());
-    worker.childStarted(shared_from_this(), fds, false, false);
-
-    return rpAccept;
-}
-#endif
-
-
 int childEntry(void * arg)
 {
     ((LocalDerivationGoal *) arg)->runChild();
@@ -1202,43 +343,6 @@ int childEntry(void * arg)
 }
 
 
-#if 0
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-#endif
-
 static std::once_flag dns_resolve_flag;
 
 static void preloadNSS() {
@@ -3688,52 +2792,6 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
 }
 
 
-#if 0
-Path DerivationGoal::openLogFile()
-{
-    logSize = 0;
-
-    if (!settings.keepLog) return "";
-
-    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
-
-    /* Create a log file. */
-    Path logDir;
-    if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
-        logDir = localStore->logDir;
-    else
-        logDir = settings.nixLogDir;
-    Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
-    createDirs(dir);
-
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
-        settings.compressLog ? ".bz2" : "");
-
-    fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
-
-    logFileSink = std::make_shared<FdSink>(fdLogFile.get());
-
-    if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
-    else
-        logSink = logFileSink;
-
-    return logFileName;
-}
-
-
-void DerivationGoal::closeLogFile()
-{
-    auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
-    if (logSink2) logSink2->finish();
-    if (logFileSink) logFileSink->flush();
-    logSink = logFileSink = 0;
-    fdLogFile = -1;
-}
-#endif
-
-
 void LocalDerivationGoal::deleteTmpDir(bool force)
 {
     if (tmpDir != "") {
@@ -3757,126 +2815,6 @@ bool LocalDerivationGoal::isReadDesc(int fd)
 }
 
 
-#if 0
-void DerivationGoal::handleChildOutput(int fd, const string & data)
-{
-    if (isReadDesc(fd))
-    {
-        logSize += data.size();
-        if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            killChild();
-            done(
-                BuildResult::LogLimitExceeded,
-                Error("%s killed after writing more than %d bytes of log output",
-                    getName(), settings.maxLogSize));
-            return;
-        }
-
-        for (auto c : data)
-            if (c == '\r')
-                currentLogLinePos = 0;
-            else if (c == '\n')
-                flushLine();
-            else {
-                if (currentLogLinePos >= currentLogLine.size())
-                    currentLogLine.resize(currentLogLinePos + 1);
-                currentLogLine[currentLogLinePos++] = c;
-            }
-
-        if (logSink) (*logSink)(data);
-    }
-
-    if (hook && fd == hook->fromHook.readSide.get()) {
-        for (auto c : data)
-            if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
-                currentHookLine.clear();
-            } else
-                currentHookLine += c;
-    }
-}
-
-
-void DerivationGoal::handleEOF(int fd)
-{
-    if (!currentLogLine.empty()) flushLine();
-    worker.wakeUp(shared_from_this());
-}
-
-
-void DerivationGoal::flushLine()
-{
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
-        ;
-
-    else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
-
-        act->result(resBuildLogLine, currentLogLine);
-    }
-
-    currentLogLine = "";
-    currentLogLinePos = 0;
-}
-
-
-std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        std::map<std::string, std::optional<StorePath>> res;
-        for (auto & [name, output] : drv->outputs)
-            res.insert_or_assign(name, output.path(worker.store, drv->name, name));
-        return res;
-    } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
-    }
-}
-
-OutputPathMap DerivationGoal::queryDerivationOutputMap()
-{
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
-        OutputPathMap res;
-        for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
-            res.insert_or_assign(name, *output.second);
-        return res;
-    } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
-    }
-}
-
-
-void DerivationGoal::checkPathValidity()
-{
-    bool checkHash = buildMode == bmRepair;
-    for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput & info = initialOutputs.at(i.first);
-        info.wanted = wantOutput(i.first, wantedOutputs);
-        if (i.second) {
-            auto outputPath = *i.second;
-            info.known = {
-                .path = outputPath,
-                .status = !worker.store.isValidPath(outputPath)
-                    ? PathStatus::Absent
-                    : !checkHash || worker.pathContentsGood(outputPath)
-                    ? PathStatus::Valid
-                    : PathStatus::Corrupt,
-            };
-        }
-        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-            if (auto real = worker.store.queryRealisation(
-                    DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) {
-                info.known = {
-                    .path = real->outPath,
-                    .status = PathStatus::Valid,
-                };
-            }
-        }
-    }
-}
-#endif
-
-
 StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
 {
     return worker.store.makeStorePath(
@@ -3893,32 +2831,4 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
 }
 
 
-#if 0
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
-{
-    result.status = status;
-    if (ex)
-        result.errorMsg = ex->what();
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
-    if (result.status == BuildResult::TimedOut)
-        worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
-        worker.permanentFailure = true;
-
-    mcExpectedBuilds.reset();
-    mcRunningBuilds.reset();
-
-    if (result.success()) {
-        if (status == BuildResult::Built)
-            worker.doneBuilds++;
-    } else {
-        if (status != BuildResult::DependencyFailed)
-            worker.failedBuilds++;
-    }
-
-    worker.updateProgress();
-}
-#endif
-
-
 }
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index f7994113e..a2b386a72 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -9,44 +9,6 @@ struct LocalDerivationGoal : public DerivationGoal
 {
     LocalStore & getLocalStore();
 
-#if 0
-    /* Whether to use an on-disk .drv file. */
-    bool useDerivation;
-
-    /* The path of the derivation. */
-    StorePath drvPath;
-
-    /* The path of the corresponding resolved derivation */
-    std::optional<BasicDerivation> resolvedDrv;
-
-    /* The specific outputs that we need to build.  Empty means all of
-       them. */
-    StringSet wantedOutputs;
-
-    /* Whether additional wanted outputs have been added. */
-    bool needRestart = false;
-
-    /* Whether to retry substituting the outputs after building the
-       inputs. */
-    bool retrySubstitution;
-
-    /* The derivation stored at drvPath. */
-    std::unique_ptr<Derivation> drv;
-
-    std::unique_ptr<ParsedDerivation> parsedDrv;
-
-    /* The remainder is state held during the build. */
-
-    /* Locks on (fixed) output paths. */
-    PathLocks outputLocks;
-
-    /* All input paths (that is, the union of FS closures of the
-       immediate input paths). */
-    StorePathSet inputPaths;
-
-    std::map<std::string, InitialOutput> initialOutputs;
-#endif
-
     /* User selected for running the builder. */
     std::unique_ptr<UserLock> buildUser;
 
@@ -59,23 +21,6 @@ struct LocalDerivationGoal : public DerivationGoal
     /* The path of the temporary directory in the sandbox. */
     Path tmpDirInSandbox;
 
-#if 0
-    /* File descriptor for the log file. */
-    AutoCloseFD fdLogFile;
-    std::shared_ptr<BufferedSink> logFileSink, logSink;
-
-    /* Number of bytes received from the builder's stdout/stderr. */
-    unsigned long logSize;
-
-    /* The most recent log lines. */
-    std::list<std::string> logTail;
-
-    std::string currentLogLine;
-    size_t currentLogLinePos = 0; // to handle carriage return
-
-    std::string currentHookLine;
-#endif
-
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
 
@@ -90,11 +35,6 @@ struct LocalDerivationGoal : public DerivationGoal
        namespace. */
     bool usingUserNamespace = true;
 
-#if 0
-    /* The build hook. */
-    std::unique_ptr<HookInstance> hook;
-#endif
-
     /* Whether we're currently doing a chroot build. */
     bool useChroot = false;
 
@@ -103,19 +43,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* RAII object to delete the chroot directory. */
     std::shared_ptr<AutoDelete> autoDelChroot;
 
-#if 0
-    /* The sort of derivation we are building. */
-    DerivationType derivationType;
-#endif
-
     /* Whether to run the build in a private network namespace. */
     bool privateNetwork = false;
 
-#if 0
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-#endif
-
     /* Stuff we need to pass to initChild(). */
     struct ChrootPath {
         Path source;
@@ -155,34 +85,11 @@ struct LocalDerivationGoal : public DerivationGoal
      */
     OutputPathMap scratchOutputs;
 
-#if 0
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-#endif
-
     /* If we're repairing without a chroot, there may be outputs that
        are valid but corrupt.  So we redirect these outputs to
        temporary paths. */
     StorePathSet redirectedBadOutputs;
 
-#if 0
-    BuildResult result;
-
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-#endif
-
     /* Path registration info from the previous round, if we're
        building multiple times. Since this contains the hash, it
        allows us to compare whether two rounds produced the same
@@ -194,20 +101,6 @@ struct LocalDerivationGoal : public DerivationGoal
 
     const static Path homeDir;
 
-#if 0
-    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
-
-    std::unique_ptr<Activity> act;
-
-    /* Activity that denotes waiting for a lock. */
-    std::unique_ptr<Activity> actLock;
-
-    std::map<ActivityId, Activity> builderActivities;
-
-    /* The remote machine on which we're building. */
-    std::string machineName;
-#endif
-
     /* The recursive Nix daemon socket. */
     AutoCloseFD daemonSocket;
 
@@ -238,37 +131,8 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Whether we need to perform hash rewriting if there are valid output paths. */
     bool needsHashRewrite();
 
-#if 0
-    void timedOut(Error && ex) override;
-
-    string key() override;
-
-    void work() override;
-
-    /* Add wanted outputs to an already existing derivation goal. */
-    void addWantedOutputs(const StringSet & outputs);
-
-    BuildResult getResult() { return result; }
-
-    /* The states. */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-#endif
+    /* The additional states. */
     void tryLocalBuild() override;
-#if 0
-    void buildDone();
-
-    void resolvedFinished();
-
-    /* Is the build hook willing to perform the build? */
-    HookReply tryBuildHook();
-#endif
 
     /* Start building a derivation. */
     void startBuilder();
@@ -307,14 +171,6 @@ struct LocalDerivationGoal : public DerivationGoal
        '{allowed,disallowed}{References,Requisites}' attributes). */
     void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
 
-#if 0
-    /* Open a log file and a pipe to it. */
-    Path openLogFile();
-
-    /* Close the log file. */
-    void closeLogFile();
-#endif
-
     /* Close the read side of the logger pipe. */
     void closeReadPipes() override;
 
@@ -328,26 +184,9 @@ struct LocalDerivationGoal : public DerivationGoal
 
     bool isReadDesc(int fd) override;
 
-
     /* Delete the temporary directory, if we have one. */
     void deleteTmpDir(bool force);
 
-#if 0
-    /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
-    void handleEOF(int fd) override;
-    void flushLine();
-
-    /* Wrappers around the corresponding Store methods that first consult the
-       derivation.  This is currently needed because when there is no drv file
-       there also is no DB entry. */
-    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
-    OutputPathMap queryDerivationOutputMap();
-
-    /* Return the set of (in)valid paths. */
-    void checkPathValidity();
-#endif
-
     /* Forcibly kill the child process, if any. */
     void killChild() override;
 
@@ -360,18 +199,6 @@ struct LocalDerivationGoal : public DerivationGoal
     /* FIXME add option to randomize, so we can audit whether our
        rewrites caught everything */
     StorePath makeFallbackPath(std::string_view outputName);
-
-#if 0
-    void repairClosure();
-
-    void started();
-
-    void done(
-        BuildResult::Status status,
-        std::optional<Error> ex = {});
-
-    StorePathSet exportReferences(const StorePathSet & storePaths);
-#endif
 };
 
 }

From 553b79f8c980fde70fe186ee4980b2d12e27d756 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 24 Feb 2021 17:38:38 +0000
Subject: [PATCH 778/882] Remove unused `redirectedBadOutputs`

---
 src/libstore/build/local-derivation-goal.cc | 4 ----
 src/libstore/build/local-derivation-goal.hh | 5 -----
 2 files changed, 9 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 3a0616864..9c2f1dda6 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -460,10 +460,6 @@ void LocalDerivationGoal::startBuilder()
                 makeFallbackPath(status.known->path);
         scratchOutputs.insert_or_assign(outputName, scratchPath);
 
-        /* A non-removed corrupted path needs to be stored here, too */
-        if (buildMode == bmRepair && !status.known->isValid())
-            redirectedBadOutputs.insert(status.known->path);
-
         /* Substitute output placeholders with the scratch output paths.
            We'll use during the build. */
         inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index a2b386a72..4bbf27a1b 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -85,11 +85,6 @@ struct LocalDerivationGoal : public DerivationGoal
      */
     OutputPathMap scratchOutputs;
 
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
     /* Path registration info from the previous round, if we're
        building multiple times. Since this contains the hash, it
        allows us to compare whether two rounds produced the same

From 5b42e5b1771061de50575b33eeeda56f40f216f2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 Feb 2021 16:29:19 +0000
Subject: [PATCH 779/882] Restore now-working
 build-remote-content-addressed-fixed test

This was

 - Added in dbf96e10ecc75410c9db798f208f8a8310842a4f.

 - Commented out in 07975979aae4e7729ae13ffeb7390d07d71ad4bd, which I
   believe only reached master by mistake.

 - Deleted in c32168c9bc161e0c9cea027853895971699510cb, when
   `tests/build-hook-ca.nix` was reused for a new test.

But the test works, and we ought to have it.
---
 tests/build-hook-ca-fixed.nix                 | 56 +++++++++++++++++++
 ...hook-ca.nix => build-hook-ca-floating.nix} |  0
 tests/build-remote-content-addressed-fixed.sh |  5 ++
 ...build-remote-content-addressed-floating.sh |  2 +-
 tests/local.mk                                |  1 +
 5 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 tests/build-hook-ca-fixed.nix
 rename tests/{build-hook-ca.nix => build-hook-ca-floating.nix} (100%)
 create mode 100644 tests/build-remote-content-addressed-fixed.sh

diff --git a/tests/build-hook-ca-fixed.nix b/tests/build-hook-ca-fixed.nix
new file mode 100644
index 000000000..ec7171ac9
--- /dev/null
+++ b/tests/build-hook-ca-fixed.nix
@@ -0,0 +1,56 @@
+{ busybox }:
+
+with import ./config.nix;
+
+let
+
+  mkDerivation = args:
+    derivation ({
+      inherit system;
+      builder = busybox;
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      outputHashMode = "recursive";
+      outputHashAlgo = "sha256";
+    } // removeAttrs args ["builder" "meta"])
+    // { meta = args.meta or {}; };
+
+  input1 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-1";
+    buildCommand = "echo FOO > $out";
+    requiredSystemFeatures = ["foo"];
+    outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=";
+  };
+
+  input2 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-2";
+    buildCommand = "echo BAR > $out";
+    requiredSystemFeatures = ["bar"];
+    outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q=";
+  };
+
+  input3 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-3";
+    buildCommand = ''
+      read x < ${input2}
+      echo $x BAZ > $out
+    '';
+    requiredSystemFeatures = ["baz"];
+    outputHash = "sha256-daKAcPp/+BYMQsVi/YYMlCKoNAxCNDsaivwSHgQqD2s=";
+  };
+
+in
+
+  mkDerivation {
+    shell = busybox;
+    name = "build-remote";
+    buildCommand =
+      ''
+        read x < ${input1}
+        read y < ${input3}
+        echo "$x $y" > $out
+      '';
+    outputHash = "sha256-5SxbkUw6xe2l9TE1uwCvTtTDysD1vhRor38OtDF0LqQ=";
+  }
diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca-floating.nix
similarity index 100%
rename from tests/build-hook-ca.nix
rename to tests/build-hook-ca-floating.nix
diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh
new file mode 100644
index 000000000..ae7441591
--- /dev/null
+++ b/tests/build-remote-content-addressed-fixed.sh
@@ -0,0 +1,5 @@
+source common.sh
+
+file=build-hook-ca-fixed.nix
+
+source build-remote.sh
diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh
index cbb75729b..7447d92bd 100644
--- a/tests/build-remote-content-addressed-floating.sh
+++ b/tests/build-remote-content-addressed-floating.sh
@@ -1,6 +1,6 @@
 source common.sh
 
-file=build-hook-ca.nix
+file=build-hook-ca-floating.nix
 
 sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
 
diff --git a/tests/local.mk b/tests/local.mk
index 7deea9ac1..4d970d5e4 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -17,6 +17,7 @@ nix_tests = \
   linux-sandbox.sh \
   build-dry.sh \
   build-remote-input-addressed.sh \
+  build-remote-content-addressed-fixed.sh \
   build-remote-content-addressed-floating.sh \
   ssh-relay.sh \
   nar-access.sh \

From 12ec962dd8a6d8058ba11e517d74f6a07b3dc903 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Thu, 25 Feb 2021 16:12:51 -0600
Subject: [PATCH 780/882] simplify changing cachix cache for install tests

- convert cachix cache name from an env into a secret so it (along
  with the token/key) can be set once per fork
- use CACHIX_AUTH_TOKEN in addition to CACHIX_SIGNING_KEY; it looks
  like cachix will try signing key first, then auth token.
---
 .github/workflows/test.yml | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bde6106e0..2531a7d35 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,52 +8,62 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
-    env:
-      CACHIX_NAME: nix-ci
+
     steps:
     - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v12
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v8
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
-  installer:
-    if: github.event_name == 'push'
-    needs: tests
+  check_cachix:
+    name: Cachix secret present for installer tests
+    runs-on: ubuntu-latest
+    outputs:
+      secret: ${{ steps.secret.outputs.secret }}
+    steps:
+      - name: Check for Cachix secret
+        id: secret
+        env:
+          _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
+        run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
+  installer:
+    needs: [tests, check_cachix]
+    if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
     runs-on: ubuntu-latest
-    env:
-      CACHIX_NAME: nix-ci
     outputs:
       installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
     steps:
     - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@v12
     - uses: cachix/cachix-action@v8
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - id: prepare-installer
       run: scripts/prepare-installer-for-github-actions
   installer_test:
-    if: github.event_name == 'push'
-    needs: installer
+    needs: [installer, check_cachix]
+    if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
-    env:
-      CACHIX_NAME: nix-ci
     steps:
     - uses: actions/checkout@v2.3.4
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@master
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
-        install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve'
+        install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
     - run: nix-instantiate -E 'builtins.currentTime' --eval
-    
\ No newline at end of file

From bd0b0f9ab7655553f64f158d5d9a9445f5604abd Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puck.moe>
Date: Fri, 26 Feb 2021 21:48:41 +0000
Subject: [PATCH 781/882] mk: add support for CPPFLAGS

---
 mk/lib.mk      | 1 +
 mk/patterns.mk | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/mk/lib.mk b/mk/lib.mk
index a09ebaa97..6b92136cd 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -153,4 +153,5 @@ endif
 	@echo "  CFLAGS: Flags for the C compiler"
 	@echo "  CXX ($(CXX)): C++ compiler to be used"
 	@echo "  CXXFLAGS: Flags for the C++ compiler"
+	@echo "  CPPFLAGS: C preprocessor flags, used for both CC and CXX"
 	@$(print-var-help)
diff --git a/mk/patterns.mk b/mk/patterns.mk
index 7319f4cdd..86a724806 100644
--- a/mk/patterns.mk
+++ b/mk/patterns.mk
@@ -1,11 +1,11 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"
-	$(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP

From 7241fdc3d2386d256ca8870ca955b498d0ac2ff7 Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puck.moe>
Date: Fri, 26 Feb 2021 22:06:06 +0000
Subject: [PATCH 782/882] Properly propagate libseccomp linker flags

---
 Makefile.config.in    | 1 +
 src/libstore/local.mk | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 9d0500e48..3c1f01d1e 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -17,6 +17,7 @@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
+LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SHELL = @bash@
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 03c4351ac..cf0933705 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -28,7 +28,7 @@ ifeq ($(OS), SunOS)
 endif
 
 ifeq ($(HAVE_SECCOMP), 1)
-	libstore_LDFLAGS += -lseccomp
+	libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
 endif
 
 libstore_CXXFLAGS += \

From 2d7917f035c7396e87546b130317a2e5234afa36 Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puck.moe>
Date: Fri, 26 Feb 2021 21:42:51 +0000
Subject: [PATCH 783/882] Revert "Add support for building JARs from Java
 sources"

This reverts commit 259086de841d155f7951c2cc50f799a4631aa512.
---
 mk/jars.mk    | 36 ------------------------------------
 mk/lib.mk     | 12 +-----------
 mk/tracing.mk |  2 --
 3 files changed, 1 insertion(+), 49 deletions(-)
 delete mode 100644 mk/jars.mk

diff --git a/mk/jars.mk b/mk/jars.mk
deleted file mode 100644
index c8513e664..000000000
--- a/mk/jars.mk
+++ /dev/null
@@ -1,36 +0,0 @@
-define build-jar
-
-  $(1)_NAME ?= $(1)
-
-  _d := $$(strip $$($(1)_DIR))
-
-  $(1)_PATH := $$(_d)/$$($(1)_NAME).jar
-
-  $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp
-
-  _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH))
-
-  $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER)
-	@rm -rf $$($(1)_TMPDIR)
-	@mkdir -p $$($(1)_TMPDIR)
-	$$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \
-	  $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \
-	  -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH"
-	@echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest
-	$$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) .
-	@rm $$($(1)_PATH).manifest
-	@rm -rf $$($(1)_TMPDIR)
-
-  $(1)_INSTALL_DIR ?= $$(jardir)
-
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar
-
-  $$(eval $$(call install-file-as, $$($(1)_PATH), $$($(1)_INSTALL_PATH), 0644))
-
-  install: $$($(1)_INSTALL_PATH)
-
-  jars-list += $$($(1)_PATH)
-
-  clean-files += $$($(1)_PATH)
-
-endef
diff --git a/mk/lib.mk b/mk/lib.mk
index a09ebaa97..6a1c465b6 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -31,7 +31,6 @@ libdir ?= $(prefix)/lib
 bindir ?= $(prefix)/bin
 libexecdir ?= $(prefix)/libexec
 datadir ?= $(prefix)/share
-jardir ?= $(datadir)/java
 localstatedir ?= $(prefix)/var
 sysconfdir ?= $(prefix)/etc
 mandir ?= $(prefix)/share/man
@@ -74,7 +73,6 @@ BUILD_DEBUG ?= 1
 ifeq ($(BUILD_DEBUG), 1)
   GLOBAL_CFLAGS += -g
   GLOBAL_CXXFLAGS += -g
-  GLOBAL_JAVACFLAGS += -g
 endif
 
 
@@ -84,7 +82,6 @@ include mk/clean.mk
 include mk/install.mk
 include mk/libraries.mk
 include mk/programs.mk
-include mk/jars.mk
 include mk/patterns.mk
 include mk/templates.mk
 include mk/tests.mk
@@ -102,7 +99,6 @@ $(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
 # Instantiate stuff.
 $(foreach lib, $(libraries), $(eval $(call build-library,$(lib))))
 $(foreach prog, $(programs), $(eval $(call build-program,$(prog))))
-$(foreach jar, $(jars), $(eval $(call build-jar,$(jar))))
 $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(bindir))))
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
@@ -113,7 +109,7 @@ $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/
 
 .PHONY: default all man help
 
-all: $(programs-list) $(libs-list) $(jars-list) $(man-pages)
+all: $(programs-list) $(libs-list) $(man-pages)
 
 man: $(man-pages)
 
@@ -137,12 +133,6 @@ ifdef libs-list
 	@echo "The following libraries can be built:"
 	@echo ""
 	@for i in $(libs-list); do echo "  $$i"; done
-endif
-ifdef jars-list
-	@echo ""
-	@echo "The following JARs can be built:"
-	@echo ""
-	@for i in $(jars-list); do echo "  $$i"; done
 endif
 	@echo ""
 	@echo "The following variables control the build:"
diff --git a/mk/tracing.mk b/mk/tracing.mk
index 54c77ab60..1fc5573d7 100644
--- a/mk/tracing.mk
+++ b/mk/tracing.mk
@@ -8,8 +8,6 @@ ifeq ($(V), 0)
   trace-ld      = @echo "  LD    " $@;
   trace-ar      = @echo "  AR    " $@;
   trace-install = @echo "  INST  " $@;
-  trace-javac   = @echo "  JAVAC " $@;
-  trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
   trace-test    = @echo "  TEST  " $@;
 

From ae1441e5488a0e1608851b329358eb390a08ac27 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 27 Feb 2021 05:23:14 +0000
Subject: [PATCH 784/882] Fix testing fixed-output derivations in double
 sandboxes

What happened was that Nix was trying to unconditionally mount these
paths in fixed-output derivations, but since the outer derivation was
pure, those paths did not exist. The solution is to only mount those
paths when they exist.
---
 src/libstore/build/local-derivation-goal.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 9c2f1dda6..90731d98d 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -287,7 +287,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
        So instead, check if the disk is (nearly) full now.  If
        so, we don't mark this build as a permanent failure. */
 #if HAVE_STATVFS
-	{
+    {
         auto & localStore = getLocalStore();
         uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
         struct statvfs st;
@@ -297,7 +297,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         if (statvfs(tmpDir.c_str(), &st) == 0 &&
             (uint64_t) st.f_bavail * st.f_bsize < required)
             diskFull = true;
-	}
+    }
 #endif
 
     deleteTmpDir(false);
@@ -1703,18 +1703,18 @@ void LocalDerivationGoal::runChild()
                network, so give them access to /etc/resolv.conf and so
                on. */
             if (derivationIsImpure(derivationType)) {
-                ss.push_back("/etc/resolv.conf");
-
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
                 // potential impurities introduced in fixed-outputs.
                 writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
 
-                ss.push_back("/etc/services");
-                ss.push_back("/etc/hosts");
-                if (pathExists("/var/run/nscd/socket"))
-                    ss.push_back("/var/run/nscd/socket");
+                /* N.B. it is realistic that these paths might not exist. It
+                   happens when testing Nix building fixed-output derivations
+                   within a pure derivation. */
+                for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
+                    if (pathExists(path))
+                        ss.push_back(path);
             }
 
             for (auto & i : ss) dirsInChroot.emplace(i, i);

From 4bbd80c5366711b8f1b5ad108ba22206d3bee783 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 12 Feb 2021 21:50:50 +0000
Subject: [PATCH 785/882] Throw error for derivation goal with bogus wanted
 output

---
 src/libstore/build/derivation-goal.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index c29237f5c..530f8829a 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1243,9 +1243,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
 void DerivationGoal::checkPathValidity()
 {
     bool checkHash = buildMode == bmRepair;
+    auto wantedOutputsLeft = wantedOutputs;
     for (auto & i : queryPartialDerivationOutputMap()) {
         InitialOutput & info = initialOutputs.at(i.first);
         info.wanted = wantOutput(i.first, wantedOutputs);
+        if (info.wanted)
+            wantedOutputsLeft.erase(i.first);
         if (i.second) {
             auto outputPath = *i.second;
             info.known = {
@@ -1267,6 +1270,11 @@ void DerivationGoal::checkPathValidity()
             }
         }
     }
+    // If we requested all the outputs via the empty set, we are always fine.
+    // If we requested specific elements, the loop above removes all the valid
+    // ones, so any that are left must be invalid.
+    if (!wantedOutputsLeft.empty())
+        throw UsageError("some wanted outputs are not provided by the derivation: %s", concatStringsSep(", ", wantedOutputsLeft));
 }
 
 

From 259d6778efd865ccd3b5fbf4f3a29002a7d58d93 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 9 Nov 2020 16:04:18 +0100
Subject: [PATCH 786/882] Move the CA tests to a sub-directory

Requires a slight update to the test infra to work properly, but
having the possibility to group tests that way makes the whole thing
quite cleaner imho
---
 mk/run_test.sh                                          | 2 +-
 mk/tests.mk                                             | 2 +-
 tests/{content-addressed.sh => ca/build.sh}             | 2 ++
 tests/ca/common.sh                                      | 1 +
 tests/{ => ca}/content-addressed.nix                    | 2 +-
 tests/{nix-copy-content-addressed.sh => ca/nix-copy.sh} | 0
 tests/common.sh.in                                      | 2 +-
 tests/local.mk                                          | 6 +++---
 8 files changed, 10 insertions(+), 7 deletions(-)
 rename tests/{content-addressed.sh => ca/build.sh} (98%)
 create mode 100644 tests/ca/common.sh
 rename tests/{ => ca}/content-addressed.nix (98%)
 rename tests/{nix-copy-content-addressed.sh => ca/nix-copy.sh} (100%)

diff --git a/mk/run_test.sh b/mk/run_test.sh
index 6af5b070a..3783d3bf7 100755
--- a/mk/run_test.sh
+++ b/mk/run_test.sh
@@ -14,7 +14,7 @@ if [ -t 1 ]; then
     yellow=""
     normal=""
 fi
-(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
 log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
 status=$?
 if [ $status -eq 0 ]; then
diff --git a/mk/tests.mk b/mk/tests.mk
index c1e140bac..21bdc5748 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -8,7 +8,7 @@ define run-install-test
 
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
+	@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
 
 endef
 
diff --git a/tests/content-addressed.sh b/tests/ca/build.sh
similarity index 98%
rename from tests/content-addressed.sh
rename to tests/ca/build.sh
index 7e32e1f28..35bf1dcf7 100644
--- a/tests/content-addressed.sh
+++ b/tests/ca/build.sh
@@ -61,7 +61,9 @@ testNixCommand () {
 
 # Disabled until we have it properly working
 # testRemoteCache
+clearStore
 testDeterministicCA
+clearStore
 testCutoff
 testGC
 testNixCommand
diff --git a/tests/ca/common.sh b/tests/ca/common.sh
new file mode 100644
index 000000000..e083d873c
--- /dev/null
+++ b/tests/ca/common.sh
@@ -0,0 +1 @@
+source ../common.sh
diff --git a/tests/content-addressed.nix b/tests/ca/content-addressed.nix
similarity index 98%
rename from tests/content-addressed.nix
rename to tests/ca/content-addressed.nix
index 61079176f..e5b1c4de3 100644
--- a/tests/content-addressed.nix
+++ b/tests/ca/content-addressed.nix
@@ -1,4 +1,4 @@
-with import ./config.nix;
+with import ../config.nix;
 
 { seed ? 0 }:
 # A simple content-addressed derivation.
diff --git a/tests/nix-copy-content-addressed.sh b/tests/ca/nix-copy.sh
similarity index 100%
rename from tests/nix-copy-content-addressed.sh
rename to tests/ca/nix-copy.sh
diff --git a/tests/common.sh.in b/tests/common.sh.in
index e3bcab507..de44a4da4 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -11,7 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
-export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
+export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
 unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
diff --git a/tests/local.mk b/tests/local.mk
index 7deea9ac1..07cfd7a50 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -38,10 +38,10 @@ nix_tests = \
   recursive.sh \
   describe-stores.sh \
   flakes.sh \
-  content-addressed.sh \
-  nix-copy-content-addressed.sh \
   build.sh \
-  compute-levels.sh
+  compute-levels.sh \
+  ca/build.sh \
+  ca/nix-copy.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From 5d1c05b07561c841c68eb3ff9698ce9d2355fe41 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 9 Nov 2020 13:47:06 +0100
Subject: [PATCH 787/882] SubstitutionGoal -> PathSubstitutionGoal

To prepare for the upcoming DrvOutputSubstitutionGoal
---
 src/libstore/build/derivation-goal.cc   |  8 +++----
 src/libstore/build/entry-points.cc      |  8 +++----
 src/libstore/build/substitution-goal.cc | 32 ++++++++++++-------------
 src/libstore/build/substitution-goal.hh |  9 +++----
 src/libstore/build/worker.cc            | 12 +++++-----
 src/libstore/build/worker.hh            | 12 +++++-----
 src/libstore/local-store.hh             |  2 ++
 7 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index c29237f5c..7b97e575a 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -170,7 +170,7 @@ void DerivationGoal::getDerivation()
         return;
     }
 
-    addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath)));
+    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
 
     state = &DerivationGoal::loadDerivation;
 }
@@ -253,7 +253,7 @@ void DerivationGoal::haveDerivation()
                 /* Nothing to wait for; tail call */
                 return DerivationGoal::gaveUpOnSubstitution();
             }
-            addWaitee(upcast_goal(worker.makeSubstitutionGoal(
+            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
                 status.known->path,
                 buildMode == bmRepair ? Repair : NoRepair,
                 getDerivationCA(*drv))));
@@ -337,7 +337,7 @@ void DerivationGoal::gaveUpOnSubstitution()
         if (!settings.useSubstitutes)
             throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
                 worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(upcast_goal(worker.makeSubstitutionGoal(i)));
+        addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i)));
     }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -388,7 +388,7 @@ void DerivationGoal::repairClosure()
             worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
-            addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
+            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
         else
             addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
     }
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 01a564aba..686364440 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
         if (path.path.isDerivation())
             goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
         else
-            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
+            goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
     }
 
     worker.run(goals);
@@ -31,7 +31,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
-            else if (auto i2 = dynamic_cast<SubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
+            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
         }
     }
 
@@ -90,7 +90,7 @@ void Store::ensurePath(const StorePath & path)
     if (isValidPath(path)) return;
 
     Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path);
+    GoalPtr goal = worker.makePathSubstitutionGoal(path);
     Goals goals = {goal};
 
     worker.run(goals);
@@ -108,7 +108,7 @@ void Store::ensurePath(const StorePath & path)
 void LocalStore::repairPath(const StorePath & path)
 {
     Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
+    GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
     Goals goals = {goal};
 
     worker.run(goals);
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index c4b0de78d..5d88b8758 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -5,20 +5,20 @@
 
 namespace nix {
 
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
     : Goal(worker)
     , storePath(storePath)
     , repair(repair)
     , ca(ca)
 {
-    state = &SubstitutionGoal::init;
+    state = &PathSubstitutionGoal::init;
     name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
     trace("created");
     maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
 }
 
 
-SubstitutionGoal::~SubstitutionGoal()
+PathSubstitutionGoal::~PathSubstitutionGoal()
 {
     try {
         if (thr.joinable()) {
@@ -32,13 +32,13 @@ SubstitutionGoal::~SubstitutionGoal()
 }
 
 
-void SubstitutionGoal::work()
+void PathSubstitutionGoal::work()
 {
     (this->*state)();
 }
 
 
-void SubstitutionGoal::init()
+void PathSubstitutionGoal::init()
 {
     trace("init");
 
@@ -59,7 +59,7 @@ void SubstitutionGoal::init()
 }
 
 
-void SubstitutionGoal::tryNext()
+void PathSubstitutionGoal::tryNext()
 {
     trace("trying next substituter");
 
@@ -154,16 +154,16 @@ void SubstitutionGoal::tryNext()
        paths referenced by this one. */
     for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
+            addWaitee(worker.makePathSubstitutionGoal(i));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         referencesValid();
     else
-        state = &SubstitutionGoal::referencesValid;
+        state = &PathSubstitutionGoal::referencesValid;
 }
 
 
-void SubstitutionGoal::referencesValid()
+void PathSubstitutionGoal::referencesValid()
 {
     trace("all references realised");
 
@@ -177,12 +177,12 @@ void SubstitutionGoal::referencesValid()
         if (i != storePath) /* ignore self-references */
             assert(worker.store.isValidPath(i));
 
-    state = &SubstitutionGoal::tryToRun;
+    state = &PathSubstitutionGoal::tryToRun;
     worker.wakeUp(shared_from_this());
 }
 
 
-void SubstitutionGoal::tryToRun()
+void PathSubstitutionGoal::tryToRun()
 {
     trace("trying to run");
 
@@ -221,11 +221,11 @@ void SubstitutionGoal::tryToRun()
 
     worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
 
-    state = &SubstitutionGoal::finished;
+    state = &PathSubstitutionGoal::finished;
 }
 
 
-void SubstitutionGoal::finished()
+void PathSubstitutionGoal::finished()
 {
     trace("substitute finished");
 
@@ -249,7 +249,7 @@ void SubstitutionGoal::finished()
         }
 
         /* Try the next substitute. */
-        state = &SubstitutionGoal::tryNext;
+        state = &PathSubstitutionGoal::tryNext;
         worker.wakeUp(shared_from_this());
         return;
     }
@@ -278,12 +278,12 @@ void SubstitutionGoal::finished()
 }
 
 
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
+void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
 {
 }
 
 
-void SubstitutionGoal::handleEOF(int fd)
+void PathSubstitutionGoal::handleEOF(int fd)
 {
     if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index dee2cecbf..3b3cb7e32 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -8,7 +8,7 @@ namespace nix {
 
 class Worker;
 
-struct SubstitutionGoal : public Goal
+struct PathSubstitutionGoal : public Goal
 {
     /* The store path that should be realised through a substitute. */
     StorePath storePath;
@@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal
     std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
         maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
 
-    typedef void (SubstitutionGoal::*GoalState)();
+    typedef void (PathSubstitutionGoal::*GoalState)();
     GoalState state;
 
     /* Content address for recomputing store path */
     std::optional<ContentAddress> ca;
 
-    SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
-    ~SubstitutionGoal();
+public:
+    PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+    ~PathSubstitutionGoal();
 
     void timedOut(Error && ex) override { abort(); };
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index b2223c3b6..619b1d69c 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -78,12 +78,12 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
 }
 
 
-std::shared_ptr<SubstitutionGoal> Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
 {
-    std::weak_ptr<SubstitutionGoal> & goal_weak = substitutionGoals[path];
+    std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
     auto goal = goal_weak.lock(); // FIXME
     if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
+        goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
         goal_weak = goal;
         wakeUp(goal);
     }
@@ -109,7 +109,7 @@ void Worker::removeGoal(GoalPtr goal)
 {
     if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
         nix::removeGoal(drvGoal, derivationGoals);
-    else if (auto subGoal = std::dynamic_pointer_cast<SubstitutionGoal>(goal))
+    else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
     else
         assert(false);
@@ -217,7 +217,7 @@ void Worker::run(const Goals & _topGoals)
         topGoals.insert(i);
         if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
             topPaths.push_back({goal->drvPath, goal->wantedOutputs});
-        } else if (auto goal = dynamic_cast<SubstitutionGoal *>(i.get())) {
+        } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
             topPaths.push_back({goal->storePath});
         }
     }
@@ -471,7 +471,7 @@ void Worker::markContentsGood(const StorePath & path)
 }
 
 
-GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal) {
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
     return subGoal;
 }
 
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 82e711191..42acf8542 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -12,18 +12,18 @@ namespace nix {
 
 /* Forward definition. */
 struct DerivationGoal;
-struct SubstitutionGoal;
+struct PathSubstitutionGoal;
 
 /* Workaround for not being able to declare a something like
 
-     class SubstitutionGoal : public Goal;
+     class PathSubstitutionGoal : public Goal;
 
    even when Goal is a complete type.
 
    This is still a static cast. The purpose of exporting it is to define it in
-   a place where `SubstitutionGoal` is concrete, and use it in a place where it
+   a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
    is opaque. */
-GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
@@ -72,7 +72,7 @@ private:
     /* Maps used to prevent multiple instantiations of a goal for the
        same derivation / path. */
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
-    std::map<StorePath, std::weak_ptr<SubstitutionGoal>> substitutionGoals;
+    std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
 
     /* Goals waiting for busy paths to be unlocked. */
     WeakGoals waitingForAnyGoal;
@@ -146,7 +146,7 @@ public:
         const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
 
     /* substitution goal */
-    std::shared_ptr<SubstitutionGoal> makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+    std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
 
     /* Remove a dead goal. */
     void removeGoal(GoalPtr goal);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 03bb0218d..fc67f215a 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -281,7 +281,9 @@ private:
     void createUser(const std::string & userName, uid_t userId) override;
 
     friend struct LocalDerivationGoal;
+    friend struct PathSubstitutionGoal;
     friend struct SubstitutionGoal;
+    friend struct DerivationGoal;
 };
 
 

From df9d4f88d5aed0aa4ed67eb012e9f260550b7200 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 9 Nov 2020 15:40:10 +0100
Subject: [PATCH 788/882] Allow substituting drv outputs when building

---
 src/libstore/build/derivation-goal.cc         | 25 +++--
 .../build/drv-output-substitution-goal.cc     | 95 +++++++++++++++++++
 .../build/drv-output-substitution-goal.hh     | 50 ++++++++++
 src/libstore/build/worker.cc                  | 22 ++++-
 src/libstore/build/worker.hh                  |  5 +
 5 files changed, 185 insertions(+), 12 deletions(-)
 create mode 100644 src/libstore/build/drv-output-substitution-goal.cc
 create mode 100644 src/libstore/build/drv-output-substitution-goal.hh

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 7b97e575a..7dcd2a6eb 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -246,17 +246,22 @@ void DerivationGoal::haveDerivation()
        through substitutes.  If that doesn't work, we'll build
        them. */
     if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
+        for (auto & [outputName, status] : initialOutputs) {
             if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv))));
+            if (!status.known)
+                addWaitee(
+                    upcast_goal(
+                        worker.makeDrvOutputSubstitutionGoal(
+                            DrvOutput{status.outputHash, outputName},
+                            buildMode == bmRepair ? Repair : NoRepair
+                        )
+                    )
+                );
+            else
+                addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
+                    status.known->path,
+                    buildMode == bmRepair ? Repair : NoRepair,
+                    getDerivationCA(*drv))));
         }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
new file mode 100644
index 000000000..a5ac4c49d
--- /dev/null
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -0,0 +1,95 @@
+#include "drv-output-substitution-goal.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+
+namespace nix {
+
+DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+    : Goal(worker)
+    , id(id)
+{
+    state = &DrvOutputSubstitutionGoal::init;
+    name = fmt("substitution of '%s'", id.to_string());
+    trace("created");
+}
+
+
+void DrvOutputSubstitutionGoal::init()
+{
+    trace("init");
+    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    tryNext();
+}
+
+void DrvOutputSubstitutionGoal::tryNext()
+{
+    trace("Trying next substituter");
+
+    if (subs.size() == 0) {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+        debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
+
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+
+        if (substituterFailed) {
+            worker.failedSubstitutions++;
+            worker.updateProgress();
+        }
+
+        return;
+    }
+
+    auto sub = subs.front();
+    subs.pop_front();
+
+    // FIXME: Make async
+    outputInfo = sub->queryRealisation(id);
+    if (!outputInfo) {
+        tryNext();
+        return;
+    }
+
+    addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
+
+    if (waitees.empty()) outPathValid();
+    else state = &DrvOutputSubstitutionGoal::outPathValid;
+}
+
+void DrvOutputSubstitutionGoal::outPathValid()
+{
+    assert(outputInfo);
+    trace("Output path substituted");
+
+    if (nrFailed > 0) {
+        debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+        return;
+    }
+
+    worker.store.registerDrvOutput(*outputInfo);
+    finished();
+}
+
+void DrvOutputSubstitutionGoal::finished()
+{
+    trace("finished");
+    amDone(ecSuccess);
+}
+
+string DrvOutputSubstitutionGoal::key()
+{
+    /* "a$" ensures substitution goals happen before derivation
+       goals. */
+    return "a$" + std::string(id.to_string());
+}
+
+void DrvOutputSubstitutionGoal::work()
+{
+    (this->*state)();
+}
+
+}
diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh
new file mode 100644
index 000000000..63ab53d89
--- /dev/null
+++ b/src/libstore/build/drv-output-substitution-goal.hh
@@ -0,0 +1,50 @@
+#pragma once
+
+#include "store-api.hh"
+#include "goal.hh"
+#include "realisation.hh"
+
+namespace nix {
+
+class Worker;
+
+// Substitution of a derivation output.
+// This is done in three steps:
+// 1. Fetch the output info from a substituter
+// 2. Substitute the corresponding output path
+// 3. Register the output info
+class DrvOutputSubstitutionGoal : public Goal {
+private:
+    // The drv output we're trying to substitue
+    DrvOutput id;
+
+    // The realisation corresponding to the given output id.
+    // Will be filled once we can get it.
+    std::optional<Realisation> outputInfo;
+
+    /* The remaining substituters. */
+    std::list<ref<Store>> subs;
+
+    /* Whether a substituter failed. */
+    bool substituterFailed = false;
+
+public:
+    DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+
+    typedef void (DrvOutputSubstitutionGoal::*GoalState)();
+    GoalState state;
+
+    void init();
+    void tryNext();
+    void outPathValid();
+    void finished();
+
+    void timedOut(Error && ex) override { abort(); };
+
+    string key() override;
+
+    void work() override;
+
+};
+
+}
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 619b1d69c..616b17e61 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -1,6 +1,7 @@
 #include "machines.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
+#include "drv-output-substitution-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
@@ -90,8 +91,20 @@ std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const Sto
     return goal;
 }
 
-template<typename G>
-static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap)
+std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+{
+    std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
+    auto goal = goal_weak.lock(); // FIXME
+    if (!goal) {
+        goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
+        goal_weak = goal;
+        wakeUp(goal);
+    }
+    return goal;
+}
+
+template<typename K, typename G>
+static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
     for (auto i = goalMap.begin();
@@ -111,6 +124,8 @@ void Worker::removeGoal(GoalPtr goal)
         nix::removeGoal(drvGoal, derivationGoals);
     else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
+    else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
+        nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
     else
         assert(false);
     if (topGoals.find(goal) != topGoals.end()) {
@@ -474,5 +489,8 @@ void Worker::markContentsGood(const StorePath & path)
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
     return subGoal;
 }
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
+    return subGoal;
+}
 
 }
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 42acf8542..918de35f6 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -4,6 +4,7 @@
 #include "lock.hh"
 #include "store-api.hh"
 #include "goal.hh"
+#include "realisation.hh"
 
 #include <future>
 #include <thread>
@@ -13,6 +14,7 @@ namespace nix {
 /* Forward definition. */
 struct DerivationGoal;
 struct PathSubstitutionGoal;
+class DrvOutputSubstitutionGoal;
 
 /* Workaround for not being able to declare a something like
 
@@ -24,6 +26,7 @@ struct PathSubstitutionGoal;
    a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
    is opaque. */
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
@@ -73,6 +76,7 @@ private:
        same derivation / path. */
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
     std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
+    std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
 
     /* Goals waiting for busy paths to be unlocked. */
     WeakGoals waitingForAnyGoal;
@@ -147,6 +151,7 @@ public:
 
     /* substitution goal */
     std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+    std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
 
     /* Remove a dead goal. */
     void removeGoal(GoalPtr goal);

From 93b5a59b674c0a29846828c7d14b434cc954f8ee Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 9 Nov 2020 16:04:43 +0100
Subject: [PATCH 789/882] Add a test for the remote caching of CA derivations

---
 tests/ca/substitute.sh | 21 +++++++++++++++++++++
 tests/local.mk         |  3 ++-
 tests/push-to-store.sh |  6 ++++--
 3 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 tests/ca/substitute.sh

diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
new file mode 100644
index 000000000..79a6ef8b1
--- /dev/null
+++ b/tests/ca/substitute.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Ensure that binary substitution works properly with ca derivations
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+export REMOTE_STORE=file://$TEST_ROOT/binary_cache
+
+buildDrvs () {
+    nix build --file ./content-addressed.nix -L --no-link "$@"
+}
+
+# Populate the remote cache
+buildDrvs --post-build-hook ../push-to-store.sh
+
+# Restart the build on an empty store, ensuring that we don't build
+clearStore
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
+
diff --git a/tests/local.mk b/tests/local.mk
index 07cfd7a50..e17555051 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -41,7 +41,8 @@ nix_tests = \
   build.sh \
   compute-levels.sh \
   ca/build.sh \
-  ca/nix-copy.sh
+  ca/nix-copy.sh \
+  ca/substitute.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh
index 6aadb916b..25352c751 100755
--- a/tests/push-to-store.sh
+++ b/tests/push-to-store.sh
@@ -1,4 +1,6 @@
 #!/bin/sh
 
-echo Pushing "$@" to "$REMOTE_STORE"
-printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs
+set -x
+
+echo Pushing "$OUT_PATHS" to "$REMOTE_STORE"
+printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs

From 9931f18c2dfff2642dea8e1a153eaaa58d7e3c8a Mon Sep 17 00:00:00 2001
From: Kjetil Orbekk <kj@orbekk.com>
Date: Sun, 21 Feb 2021 11:08:28 -0500
Subject: [PATCH 790/882] Add support for bare git repositories with git+file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Local git repositories are normally used directly instead of
cloning. This commit checks if a repo is bare and forces a
clone.

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
---
 src/libfetchers/git.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 81c647f89..4f9db1bcd 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -153,12 +153,14 @@ struct GitInputScheme : InputScheme
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const
     {
-        // Don't clone file:// URIs (but otherwise treat them the
-        // same as remote URIs, i.e. don't use the working tree or
-        // HEAD).
+        // file:// URIs are normally not cloned (but otherwise treated the
+        // same as remote URIs, i.e. we don't use the working tree or
+        // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
+        // repo, treat as a remote URI to force a clone.
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
         auto url = parseURL(getStrAttr(input.attrs, "url"));
-        bool isLocal = url.scheme == "file" && !forceHttp;
+        bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
+        bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
         return {isLocal, isLocal ? url.path : url.base};
     }
 

From 92a234322f5a46b65825c748220cef40209eeacd Mon Sep 17 00:00:00 2001
From: Kjetil Orbekk <kj@orbekk.com>
Date: Sun, 21 Feb 2021 10:41:46 -0500
Subject: [PATCH 791/882] Add test for git+file with bare repository

---
 tests/flakes.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/flakes.sh b/tests/flakes.sh
index 25ba2ac43..9747aba7a 100644
--- a/tests/flakes.sh
+++ b/tests/flakes.sh
@@ -25,6 +25,7 @@ templatesDir=$TEST_ROOT/templates
 nonFlakeDir=$TEST_ROOT/nonFlake
 flakeA=$TEST_ROOT/flakeA
 flakeB=$TEST_ROOT/flakeB
+flakeGitBare=$TEST_ROOT/flakeGitBare
 
 for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB; do
     rm -rf $repo $repo.tmp
@@ -604,6 +605,11 @@ nix flake update $flake3Dir
 [[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]]
 [[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
 
+# Test git+file with bare repo.
+rm -rf $flakeGitBare
+git clone --bare $flake1Dir $flakeGitBare
+nix build -o $TEST_ROOT/result git+file://$flakeGitBare
+
 # Test Mercurial flakes.
 rm -rf $flake5Dir
 hg init $flake5Dir

From 7ce10924c74e9e037b05558aeb5f0639df5955f6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 1 Mar 2021 15:07:09 +0000
Subject: [PATCH 792/882] Fix bad wanted output error as requested

- UsageError -> Error

- include drv path too
---
 src/libstore/build/derivation-goal.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 530f8829a..4c3bccf25 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1274,7 +1274,9 @@ void DerivationGoal::checkPathValidity()
     // If we requested specific elements, the loop above removes all the valid
     // ones, so any that are left must be invalid.
     if (!wantedOutputsLeft.empty())
-        throw UsageError("some wanted outputs are not provided by the derivation: %s", concatStringsSep(", ", wantedOutputsLeft));
+        throw Error("derivation '%s' does not have wanted outputs %s",
+            worker.store.printStorePath(drvPath),
+            concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
 }
 
 

From fc6bfb261d50102016ed812ecf9949d41fe539f7 Mon Sep 17 00:00:00 2001
From: dramforever <dramforever@live.com>
Date: Tue, 2 Mar 2021 21:56:50 +0800
Subject: [PATCH 793/882] libfetchers/tarball: Lock on effectiveUrl

Basically, if a tarball URL is used as a flake input, and the URL leads
to a redirect, the final redirect destination would be recorded as the
locked URL.

This allows tarballs under https://nixos.org/channels to be used as
flake inputs. If we, as before, lock on to the original URL it would
break every time the channel updates.
---
 src/libfetchers/fetchers.hh |  8 +++++++-
 src/libfetchers/github.cc   |  6 +++---
 src/libfetchers/tarball.cc  | 19 ++++++++++++++-----
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index a72cfafa4..c6b219c02 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -145,7 +145,13 @@ DownloadFileResult downloadFile(
     bool immutable,
     const Headers & headers = {});
 
-std::pair<Tree, time_t> downloadTarball(
+struct DownloadTarballMeta
+{
+    time_t lastModified;
+    std::string effectiveUrl;
+};
+
+std::pair<Tree, DownloadTarballMeta> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8352ef02d..3e5ad75a8 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
+        input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified));
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", uint64_t(lastModified)}
+                {"lastModified", uint64_t(meta.lastModified)}
             },
             tree.storePath,
             true);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index b8d7d2c70..bd05bb2f1 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -109,7 +109,7 @@ DownloadFileResult downloadFile(
     };
 }
 
-std::pair<Tree, time_t> downloadTarball(
+std::pair<Tree, DownloadTarballMeta> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -127,7 +127,10 @@ std::pair<Tree, time_t> downloadTarball(
     if (cached && !cached->expired)
         return {
             Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
-            getIntAttr(cached->infoAttrs, "lastModified")
+            {
+                .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")),
+                .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url),
+            },
         };
 
     auto res = downloadFile(store, url, name, immutable, headers);
@@ -152,6 +155,7 @@ std::pair<Tree, time_t> downloadTarball(
 
     Attrs infoAttrs({
         {"lastModified", uint64_t(lastModified)},
+        {"effectiveUrl", res.effectiveUrl},
         {"etag", res.etag},
     });
 
@@ -164,7 +168,10 @@ std::pair<Tree, time_t> downloadTarball(
 
     return {
         Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
-        lastModified,
+        {
+            .lastModified = lastModified,
+            .effectiveUrl = res.effectiveUrl,
+        },
     };
 }
 
@@ -223,9 +230,11 @@ struct TarballInputScheme : InputScheme
         return true;
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        Input input(_input);
+        auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false);
+        input.attrs.insert_or_assign("url", meta.effectiveUrl);
         return {std::move(tree), input};
     }
 };

From 7331da99abead2b59efcfdaf729cb1034642b630 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 5 Feb 2021 13:35:31 +0100
Subject: [PATCH 794/882] Make NIX_SHOW_STATS work with new-style commands

---
 src/libcmd/command.hh      | 2 ++
 src/libcmd/installables.cc | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index c02193924..e66c697eb 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -48,6 +48,8 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs
     ref<EvalState> getEvalState();
 
     std::shared_ptr<EvalState> evalState;
+
+    ~EvalCommand();
 };
 
 struct MixFlakeOptions : virtual Args, EvalCommand
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 4739dc974..7102f5a1a 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -280,6 +280,12 @@ ref<EvalState> EvalCommand::getEvalState()
     return ref<EvalState>(evalState);
 }
 
+EvalCommand::~EvalCommand()
+{
+    if (evalState)
+        evalState->printStats();
+}
+
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")

From 665d4ec2dac6734caff9de5b030be123cb7276ef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 Mar 2021 17:52:57 +0100
Subject: [PATCH 795/882] nix repl :doc: Don't return docs for partially
 applied primops

This gives misleading results for Nixpkgs functions like lib.toUpper.

Fixes #4596.
---
 src/libexpr/eval.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e2f2308aa..3afe2e47b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -592,10 +592,8 @@ Value & EvalState::getBuiltin(const string & name)
 
 std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
-    if (v.isPrimOp() || v.isPrimOpApp()) {
+    if (v.isPrimOp()) {
         auto v2 = &v;
-        while (v2->isPrimOpApp())
-            v2 = v2->primOpApp.left;
         if (v2->primOp->doc)
             return Doc {
                 .pos = noPos,

From e16431b4665c0362f66bace7734fed0a6c0692d5 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Thu, 4 Mar 2021 16:14:23 +0700
Subject: [PATCH 796/882] improve man page for nix.conf (builders)

---
 src/libstore/globals.hh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index a51d9c2f1..bf0767dfa 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -206,7 +206,17 @@ public:
 
     Setting<std::string> builders{
         this, "@" + nixConfDir + "/machines", "builders",
-        "A semicolon-separated list of build machines, in the format of `nix.machines`."};
+        R"(
+          A semicolon-separated list of build machines, where each machine follows this format:
+
+          {protocol}://{user}@{host} [{comma sep. systems} - {maxJobs} {speedFactor} {comma sep. features}]
+
+          Examples:
+          
+          ssh://root@builder1.com
+          
+          ssh://root@builder2.com x86_64-linux,aarch64-linux - 40 20 nixos-test,benchmark,big-parallel,kvm
+        )"};
 
     Setting<bool> buildersUseSubstitutes{
         this, false, "builders-use-substitutes",

From 6212e89bf604d61fc896f21f66908be6fbbfcc5d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 5 Mar 2021 00:49:46 +0000
Subject: [PATCH 797/882] Avoid some StorePath -> Path -> StorePath roundtrips

There were done when StorePath was defined in Rust and there were some
FFI issues. This is no longer an issue.
---
 src/libstore/misc.cc | 35 +++++++++++++++--------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index ad4dccef9..f58816ad8 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -22,55 +22,53 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
 
     Sync<State> state_(State{0, paths_, 0});
 
-    std::function<void(const Path &)> enqueue;
+    std::function<void(const StorePath &)> enqueue;
 
     std::condition_variable done;
 
-    enqueue = [&](const Path & path) -> void {
+    enqueue = [&](const StorePath & path) -> void {
         {
             auto state(state_.lock());
             if (state->exc) return;
-            if (!state->paths.insert(parseStorePath(path)).second) return;
+            if (!state->paths.insert(path).second) return;
             state->pending++;
         }
 
-        queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) {
+        queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) {
             // FIXME: calls to isValidPath() should be async
 
             try {
                 auto info = fut.get();
 
-                auto path = parseStorePath(pathS);
-
                 if (flipDirection) {
 
                     StorePathSet referrers;
                     queryReferrers(path, referrers);
                     for (auto & ref : referrers)
                         if (ref != path)
-                            enqueue(printStorePath(ref));
+                            enqueue(ref);
 
                     if (includeOutputs)
                         for (auto & i : queryValidDerivers(path))
-                            enqueue(printStorePath(i));
+                            enqueue(i);
 
                     if (includeDerivers && path.isDerivation())
                         for (auto & i : queryDerivationOutputs(path))
                             if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                                enqueue(printStorePath(i));
+                                enqueue(i);
 
                 } else {
 
                     for (auto & ref : info->references)
                         if (ref != path)
-                            enqueue(printStorePath(ref));
+                            enqueue(ref);
 
                     if (includeOutputs && path.isDerivation())
                         for (auto & i : queryDerivationOutputs(path))
-                            if (isValidPath(i)) enqueue(printStorePath(i));
+                            if (isValidPath(i)) enqueue(i);
 
                     if (includeDerivers && info->deriver && isValidPath(*info->deriver))
-                        enqueue(printStorePath(*info->deriver));
+                        enqueue(*info->deriver);
 
                 }
 
@@ -90,7 +88,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
     };
 
     for (auto & startPath : startPaths)
-        enqueue(printStorePath(startPath));
+        enqueue(startPath);
 
     {
         auto state(state_.lock());
@@ -160,13 +158,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     };
 
     auto checkOutput = [&](
-        const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_)
+        const StorePath & drvPath, ref<Derivation> drv, const StorePath & outPath, ref<Sync<DrvState>> drvState_)
     {
         if (drvState_->lock()->done) return;
 
-        auto drvPath = parseStorePath(drvPathS);
-        auto outPath = parseStorePath(outPathS);
-
         SubstitutablePathInfos infos;
         querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
 
@@ -203,7 +198,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 return;
             }
 
-            PathSet invalid;
+            StorePathSet invalid;
             /* true for regular derivations, and CA derivations for which we
                have a trust mapping for all wanted outputs. */
             auto knownOutputPaths = true;
@@ -213,7 +208,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                     break;
                 }
                 if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
-                    invalid.insert(printStorePath(*pathOpt));
+                    invalid.insert(*pathOpt);
             }
             if (knownOutputPaths && invalid.empty()) return;
 
@@ -223,7 +218,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
-                    pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
+                    pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState));
             } else
                 mustBuildDrv(path.path, *drv);
 

From 6e849e3b0a6eb46e6dc65cbd091cc829eab09a5f Mon Sep 17 00:00:00 2001
From: Bernardo Meurer <bernardo@meurer.org>
Date: Wed, 3 Mar 2021 14:46:15 -0800
Subject: [PATCH 798/882] nix-build: set execfail

When starting a nix-shell with `-i` it was previously possible for it to
silently fail in the scenario where the specified interpreter didn't
exist. This happened due to the `exec` call masking the issue.

With this change we enable `execfail`, which causes the script using
`nix-shell` as interpreter to correctly exit with code 127.

Fixes: #4598
---
 src/nix-build/nix-build.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 7b4a53919..65b85b304 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -447,6 +447,7 @@ static void main_nix_build(int argc, char * * argv)
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "
                 "unset TZ; %6%"
+                "shopt -s execfail;"
                 "%7%",
                 shellEscape(tmpDir),
                 (pure ? "" : "p=$PATH; "),

From ac8ba2eae4fc649d7a3a19815631b4d76e60d74a Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Sat, 6 Mar 2021 19:51:29 -0600
Subject: [PATCH 799/882] remove doc for obsolete --no-build-hook flag

`--no-build-hook` appears to have been removed in 25f32625e2f2a3a1e1b3a3811da82f21c3a3b880
---
 doc/manual/src/command-ref/opt-common.md | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index 9650f53f8..bc8eb6796 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -134,15 +134,6 @@ Most Nix commands accept the following command-line options:
     failure in obtaining the substitutes to lead to a full build from
     source (with the related consumption of resources).
 
-  - `--no-build-hook`  
-    Disables the build hook mechanism. This allows to ignore remote
-    builders if they are setup on the machine.
-    
-    It's useful in cases where the bandwidth between the client and the
-    remote builder is too low. In that case it can take more time to
-    upload the sources to the remote builder and fetch back the result
-    than to do the computation locally.
-
   - `--readonly-mode`  
     When this option is used, no attempt is made to open the Nix
     database. Most Nix operations do need database access, so those

From 89013bdd7ed4007871cc421315b51b7cada8edff Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 9 Mar 2021 10:11:25 +0100
Subject: [PATCH 800/882] Add a `nix realisation` command for working on
 realisations

Currently only has `nix realisation info`, more to come probably
---
 src/nix/realisation.cc      | 78 +++++++++++++++++++++++++++++++++++++
 src/nix/realisation/info.md | 15 +++++++
 2 files changed, 93 insertions(+)
 create mode 100644 src/nix/realisation.cc
 create mode 100644 src/nix/realisation/info.md

diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc
new file mode 100644
index 000000000..9ee9ccb91
--- /dev/null
+++ b/src/nix/realisation.cc
@@ -0,0 +1,78 @@
+#include "command.hh"
+#include "common-args.hh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nix;
+
+struct CmdRealisation : virtual NixMultiCommand
+{
+    CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"}))
+    { }
+
+    std::string description() override
+    {
+        return "manipulate a Nix realisation";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix realisation' requires a sub-command.");
+        command->second->prepare();
+        command->second->run();
+    }
+};
+
+static auto rCmdRealisation = registerCommand<CmdRealisation>("realisation");
+
+struct CmdRealisationInfo : RealisedPathsCommand, MixJSON
+{
+    std::string description() override
+    {
+        return "query information about one or several realisations";
+    }
+
+    std::string doc() override
+    {
+        return
+            #include "realisation/info.md"
+            ;
+    }
+
+    Category category() override { return catSecondary; }
+
+    void run(ref<Store> store, std::vector<RealisedPath> paths) override
+    {
+        settings.requireExperimentalFeature("ca-derivations");
+        if (json) {
+            nlohmann::json res = nlohmann::json::array();
+            for (auto & path : paths) {
+                nlohmann::json currentPath;
+                if (auto realisation = std::get_if<Realisation>(&path.raw))
+                    currentPath = realisation->toJSON();
+                else
+                    currentPath["opaquePath"] = store->printStorePath(path.path());
+
+                res.push_back(currentPath);
+            }
+            std::cout << res.dump();
+        }
+        else {
+            for (auto & path : paths) {
+                if (auto realisation = std::get_if<Realisation>(&path.raw)) {
+                    std::cout <<
+                        realisation->id.to_string() << " " <<
+                        store->printStorePath(realisation->outPath);
+                } else
+                    std::cout << store->printStorePath(path.path());
+
+                std::cout << std::endl;
+            }
+        }
+    }
+};
+
+static auto rCmdRealisationInfo = registerCommand2<CmdRealisationInfo>({"realisation", "info"});
diff --git a/src/nix/realisation/info.md b/src/nix/realisation/info.md
new file mode 100644
index 000000000..852240f44
--- /dev/null
+++ b/src/nix/realisation/info.md
@@ -0,0 +1,15 @@
+R"MdBoundary(
+# Description
+
+Display some informations about the given realisation
+
+# Examples
+
+Show some information about the realisation of the `hello` package:
+
+```console
+$ nix realisation info nixpkgs#hello --json
+[{"id":"sha256:3d382378a00588e064ee30be96dd0fa7e7df7cf3fbcace85a0e7b7dada1eef25!out","outPath":"fd3m7xawvrqcg98kgz5hc2vk3x9q0lh7-hello"}]
+```
+
+)MdBoundary"

From 8a0c00b85600991cdb9aa05902defec6ac44b777 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Tue, 10 Dec 2019 15:47:38 +0700
Subject: [PATCH 801/882] Use libarchive for all compression

---
 src/libstore/filetransfer.cc |   2 +-
 src/libutil/compression.cc   | 420 +++++++++--------------------------
 src/libutil/compression.hh   |  10 +-
 src/libutil/serialise.cc     |  56 ++++-
 src/libutil/serialise.hh     |   8 +
 src/libutil/tarfile.cc       | 104 ++++-----
 src/libutil/tarfile.hh       |  19 ++
 7 files changed, 241 insertions(+), 378 deletions(-)

diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 8ea5cdc9d..514ab3bf9 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -148,7 +148,7 @@ struct curlFileTransfer : public FileTransfer
         }
 
         LambdaSink finalSink;
-        std::shared_ptr<CompressionSink> decompressionSink;
+        std::shared_ptr<FinishSink> decompressionSink;
         std::optional<StringSink> errorSink;
 
         std::exception_ptr writeException;
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 986ba2976..8ba536000 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -1,10 +1,13 @@
 #include "compression.hh"
+#include "tarfile.hh"
 #include "util.hh"
 #include "finally.hh"
 #include "logging.hh"
 
 #include <lzma.h>
 #include <bzlib.h>
+#include <archive.h>
+#include <archive_entry.h>
 #include <cstdio>
 #include <cstring>
 
@@ -35,6 +38,80 @@ struct ChunkedCompressionSink : CompressionSink
     virtual void writeInternal(std::string_view data) = 0;
 };
 
+struct ArchiveDecompressionSource : Source
+{
+    std::unique_ptr<TarArchive> archive = 0;
+    Source & src;
+    ArchiveDecompressionSource(Source & src) : src(src) {}
+    ~ArchiveDecompressionSource() override {}
+    size_t read(char * data, size_t len) override {
+        struct archive_entry* ae;
+        if (!archive) {
+            archive = std::make_unique<TarArchive>(src, true);
+            this->archive->check(archive_read_next_header(this->archive->archive, &ae), "Failed to read header (%s)");
+            if (archive_filter_count(this->archive->archive) < 2) {
+                throw CompressionError("Input compression not recognized.");
+            }
+        }
+        ssize_t result = archive_read_data(this->archive->archive, data, len);
+        if (result > 0) return result;
+        if (result == 0) {
+            throw EndOfFile("reached end of compressed file");
+        }
+        this->archive->check(result, "Failed to read compressed data (%s)");
+        return result;
+    }
+};
+struct ArchiveCompressionSink : CompressionSink
+{
+    Sink & nextSink;
+    struct archive* archive;
+    ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
+        archive = archive_write_new();
+        if (!archive) throw Error("failed to initialize libarchive");
+        check(archive_write_add_filter_by_name(archive, format.c_str()), "Couldn't initialize compression (%s)");
+        check(archive_write_set_format_raw(archive));
+        if (format == "xz" && parallel) {
+            check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
+        }
+        // disable internal buffering
+        check(archive_write_set_bytes_per_block(archive, 0));
+        // disable output padding
+        check(archive_write_set_bytes_in_last_block(archive, 1));
+        open();
+    }
+    ~ArchiveCompressionSink() override {
+        if (archive) archive_write_free(archive);
+    }
+    void finish() override {
+        flush();
+        check(archive_write_close(archive));
+    }
+    void check(int err, const char *reason="Failed to compress (%s)") {
+        if (err == ARCHIVE_EOF)
+            throw EndOfFile("reached end of archive");
+        else if (err != ARCHIVE_OK)
+            throw Error(reason, archive_error_string(this->archive));
+    }
+    void write(std::string_view data) override {
+        ssize_t result = archive_write_data(archive, data.data(), data.length());
+        if (result <= 0) check(result);
+    }
+private:
+    void open() {
+        check(archive_write_open(archive, this, NULL, ArchiveCompressionSink::callback_write, NULL));
+        struct archive_entry *ae = archive_entry_new();
+        archive_entry_set_filetype(ae, AE_IFREG);
+        check(archive_write_header(archive, ae));
+        archive_entry_free(ae);
+    }
+    static ssize_t callback_write(struct archive *archive, void *_self, const void *buffer, size_t length) {
+        ArchiveCompressionSink *self = (ArchiveCompressionSink *)_self;
+        self->nextSink({(const char*)buffer, length});
+        return length;
+    }
+};
+
 struct NoneSink : CompressionSink
 {
     Sink & nextSink;
@@ -43,171 +120,6 @@ struct NoneSink : CompressionSink
     void write(std::string_view data) override { nextSink(data); }
 };
 
-struct GzipDecompressionSink : CompressionSink
-{
-    Sink & nextSink;
-    z_stream strm;
-    bool finished = false;
-    uint8_t outbuf[BUFSIZ];
-
-    GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        strm.zalloc = Z_NULL;
-        strm.zfree = Z_NULL;
-        strm.opaque = Z_NULL;
-        strm.avail_in = 0;
-        strm.next_in = Z_NULL;
-        strm.next_out = outbuf;
-        strm.avail_out = sizeof(outbuf);
-
-        // Enable gzip and zlib decoding (+32) with 15 windowBits
-        int ret = inflateInit2(&strm,15+32);
-        if (ret != Z_OK)
-            throw CompressionError("unable to initialise gzip encoder");
-    }
-
-    ~GzipDecompressionSink()
-    {
-        inflateEnd(&strm);
-    }
-
-    void finish() override
-    {
-        CompressionSink::flush();
-        write({});
-    }
-
-    void write(std::string_view data) override
-    {
-        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
-        strm.next_in = (Bytef *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            int ret = inflate(&strm,Z_SYNC_FLUSH);
-            if (ret != Z_OK && ret != Z_STREAM_END)
-                throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
-                    zError(ret), data.size(), strm.avail_in);
-
-            finished = ret == Z_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = (Bytef *) outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
-struct XzDecompressionSink : CompressionSink
-{
-    Sink & nextSink;
-    uint8_t outbuf[BUFSIZ];
-    lzma_stream strm = LZMA_STREAM_INIT;
-    bool finished = false;
-
-    XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        lzma_ret ret = lzma_stream_decoder(
-            &strm, UINT64_MAX, LZMA_CONCATENATED);
-        if (ret != LZMA_OK)
-            throw CompressionError("unable to initialise lzma decoder");
-
-        strm.next_out = outbuf;
-        strm.avail_out = sizeof(outbuf);
-    }
-
-    ~XzDecompressionSink()
-    {
-        lzma_end(&strm);
-    }
-
-    void finish() override
-    {
-        CompressionSink::flush();
-        write({});
-    }
-
-    void write(std::string_view data) override
-    {
-        strm.next_in = (const unsigned char *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
-            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw CompressionError("error %d while decompressing xz file", ret);
-
-            finished = ret == LZMA_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
-struct BzipDecompressionSink : ChunkedCompressionSink
-{
-    Sink & nextSink;
-    bz_stream strm;
-    bool finished = false;
-
-    BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        memset(&strm, 0, sizeof(strm));
-        int ret = BZ2_bzDecompressInit(&strm, 0, 0);
-        if (ret != BZ_OK)
-            throw CompressionError("unable to initialise bzip2 decoder");
-
-        strm.next_out = (char *) outbuf;
-        strm.avail_out = sizeof(outbuf);
-    }
-
-    ~BzipDecompressionSink()
-    {
-        BZ2_bzDecompressEnd(&strm);
-    }
-
-    void finish() override
-    {
-        flush();
-        write({});
-    }
-
-    void writeInternal(std::string_view data) override
-    {
-        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
-        strm.next_in = (char *) data.data();
-        strm.avail_in = data.size();
-
-        while (strm.avail_in) {
-            checkInterrupt();
-
-            int ret = BZ2_bzDecompress(&strm);
-            if (ret != BZ_OK && ret != BZ_STREAM_END)
-                throw CompressionError("error while decompressing bzip2 file");
-
-            finished = ret == BZ_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = (char *) outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
 struct BrotliDecompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
@@ -261,161 +173,32 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
 
 ref<std::string> decompress(const std::string & method, const std::string & in)
 {
-    StringSink ssink;
-    auto sink = makeDecompressionSink(method, ssink);
-    (*sink)(in);
-    sink->finish();
-    return ssink.s;
+    if (method == "br") {
+        StringSink ssink;
+        auto sink = makeDecompressionSink(method, ssink);
+        (*sink)(in);
+        sink->finish();
+        return ssink.s;
+    } else {
+        StringSource ssrc(in);
+        auto src = makeDecompressionSource(ssrc);
+        return make_ref<std::string>(src->drain());
+    }
 }
 
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
 {
     if (method == "none" || method == "")
-        return make_ref<NoneSink>(nextSink);
-    else if (method == "xz")
-        return make_ref<XzDecompressionSink>(nextSink);
-    else if (method == "bzip2")
-        return make_ref<BzipDecompressionSink>(nextSink);
-    else if (method == "gzip")
-        return make_ref<GzipDecompressionSink>(nextSink);
+        return std::make_unique<NoneSink>(nextSink);
     else if (method == "br")
-        return make_ref<BrotliDecompressionSink>(nextSink);
+        return std::make_unique<BrotliDecompressionSink>(nextSink);
     else
-        throw UnknownCompressionMethod("unknown compression method '%s'", method);
+		return sourceToSink([&](Source & source) {
+			auto decompressionSource = makeDecompressionSource(source);
+			decompressionSource->drainInto(nextSink);
+		});
 }
 
-struct XzCompressionSink : CompressionSink
-{
-    Sink & nextSink;
-    uint8_t outbuf[BUFSIZ];
-    lzma_stream strm = LZMA_STREAM_INIT;
-    bool finished = false;
-
-    XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
-    {
-        lzma_ret ret;
-        bool done = false;
-
-        if (parallel) {
-#ifdef HAVE_LZMA_MT
-            lzma_mt mt_options = {};
-            mt_options.flags = 0;
-            mt_options.timeout = 300; // Using the same setting as the xz cmd line
-            mt_options.preset = LZMA_PRESET_DEFAULT;
-            mt_options.filters = NULL;
-            mt_options.check = LZMA_CHECK_CRC64;
-            mt_options.threads = lzma_cputhreads();
-            mt_options.block_size = 0;
-            if (mt_options.threads == 0)
-                mt_options.threads = 1;
-            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
-            // number of threads.
-            ret = lzma_stream_encoder_mt(&strm, &mt_options);
-            done = true;
-#else
-            printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
-#endif
-        }
-
-        if (!done)
-            ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
-
-        if (ret != LZMA_OK)
-            throw CompressionError("unable to initialise lzma encoder");
-
-        // FIXME: apply the x86 BCJ filter?
-
-        strm.next_out = outbuf;
-        strm.avail_out = sizeof(outbuf);
-    }
-
-    ~XzCompressionSink()
-    {
-        lzma_end(&strm);
-    }
-
-    void finish() override
-    {
-        CompressionSink::flush();
-        write({});
-    }
-
-    void write(std::string_view data) override
-    {
-        strm.next_in = (const unsigned char *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
-            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw CompressionError("error %d while compressing xz file", ret);
-
-            finished = ret == LZMA_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
-struct BzipCompressionSink : ChunkedCompressionSink
-{
-    Sink & nextSink;
-    bz_stream strm;
-    bool finished = false;
-
-    BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        memset(&strm, 0, sizeof(strm));
-        int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
-        if (ret != BZ_OK)
-            throw CompressionError("unable to initialise bzip2 encoder");
-
-        strm.next_out = (char *) outbuf;
-        strm.avail_out = sizeof(outbuf);
-    }
-
-    ~BzipCompressionSink()
-    {
-        BZ2_bzCompressEnd(&strm);
-    }
-
-    void finish() override
-    {
-        flush();
-        writeInternal({});
-    }
-
-    void writeInternal(std::string_view data) override
-    {
-        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
-        strm.next_in = (char *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
-            if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
-                throw CompressionError("error %d while compressing bzip2 file", ret);
-
-            finished = ret == BZ_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = (char *) outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
 struct BrotliCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
@@ -468,15 +251,20 @@ struct BrotliCompressionSink : ChunkedCompressionSink
         }
     }
 };
+std::unique_ptr<Source> makeDecompressionSource(Source & prev) {
+    return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev));
+}
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
+    std::vector<std::string> la_supports = {
+        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
+    };
+    if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
+        return make_ref<ArchiveCompressionSink>(nextSink, method, parallel);
+    }
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
-    else if (method == "xz")
-        return make_ref<XzCompressionSink>(nextSink, parallel);
-    else if (method == "bzip2")
-        return make_ref<BzipCompressionSink>(nextSink);
     else if (method == "br")
         return make_ref<BrotliCompressionSink>(nextSink);
     else
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index dd666a4e1..192cb3e91 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,14 +8,18 @@
 
 namespace nix {
 
-struct CompressionSink : BufferedSink
+struct CompressionSink : BufferedSink, FinishSink
 {
-    virtual void finish() = 0;
+	using BufferedSink::operator ();
+	using BufferedSink::write;
+	using FinishSink::finish;
 };
 
+std::unique_ptr<Source> makeDecompressionSource(Source & prev);
+
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
 
 ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
 
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index d1a16b6ba..374b48d79 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -201,6 +201,61 @@ static DefaultStackAllocator defaultAllocatorSingleton;
 StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
 
 
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
+{
+    struct SourceToSink : FinishSink
+    {
+        typedef boost::coroutines2::coroutine<bool> coro_t;
+
+        std::function<void(Source &)> fun;
+        std::optional<coro_t::push_type> coro;
+
+        SourceToSink(std::function<void(Source &)> fun) : fun(fun)
+        {
+        }
+
+        std::string_view cur;
+
+        void operator () (std::string_view in) override
+        {
+            if (in.empty()) return;
+            cur = in;
+
+            if (!coro)
+                coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
+                    LambdaSource source([&](char *out, size_t out_len) {
+                        if (cur.empty()) {
+                            yield();
+                            if (yield.get()) {
+                                return (size_t)0;
+                            }
+                        }
+
+                        size_t n = std::min(cur.size(), out_len);
+                        memcpy(out, cur.data(), n);
+                        cur.remove_prefix(n);
+                        return n;
+                    });
+                    fun(source);
+                });
+
+            if (!*coro) { abort(); }
+
+            if (!cur.empty()) (*coro)(false);
+        }
+
+        void finish() {
+            if (!coro) return;
+            if (!*coro) abort();
+            (*coro)(true);
+            if (*coro) abort();
+        }
+    };
+
+    return std::make_unique<SourceToSink>(fun);
+}
+
+
 std::unique_ptr<Source> sinkToSource(
     std::function<void(Sink &)> fun,
     std::function<void()> eof)
@@ -212,7 +267,6 @@ std::unique_ptr<Source> sinkToSource(
         std::function<void(Sink &)> fun;
         std::function<void()> eof;
         std::optional<coro_t::pull_type> coro;
-        bool started = false;
 
         SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
             : fun(fun), eof(eof)
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5bbbc7ce3..0fe6e8332 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -25,6 +25,13 @@ struct NullSink : Sink
     { }
 };
 
+
+struct FinishSink : virtual Sink
+{
+    virtual void finish() = 0;
+};
+
+
 /* A buffered abstract sink. Warning: a BufferedSink should not be
    used from multiple threads concurrently. */
 struct BufferedSink : virtual Sink
@@ -281,6 +288,7 @@ struct ChainSource : Source
     size_t read(char * data, size_t len) override;
 };
 
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
 
 /* Convert a function that feeds data into a Sink into a Source. The
    Source executes the function as a coroutine. */
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 2da169ba7..b5e1cb4c0 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -2,83 +2,73 @@
 #include <archive_entry.h>
 
 #include "serialise.hh"
+#include "tarfile.hh"
 
 namespace nix {
+static int callback_open(struct archive *, void *self) {
+    return ARCHIVE_OK;
+}
 
-struct TarArchive {
-    struct archive * archive;
-    Source * source;
-    std::vector<unsigned char> buffer;
+static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) {
+    TarArchive *self = (TarArchive *)_self;
+    *buffer = self->buffer.data();
 
-    void check(int err, const char * reason = "failed to extract archive: %s")
-    {
+    try {
+        return self->source->read((char *) self->buffer.data(), 4096);
+    } catch (EndOfFile &) {
+        return 0;
+    } catch (std::exception &err) {
+        archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
+
+        return -1;
+    }
+}
+
+static int callback_close(struct archive *, void *self) {
+    return ARCHIVE_OK;
+}
+
+void TarArchive::check(int err, const char *reason)
+{
         if (err == ARCHIVE_EOF)
             throw EndOfFile("reached end of archive");
         else if (err != ARCHIVE_OK)
             throw Error(reason, archive_error_string(this->archive));
     }
 
-    TarArchive(Source & source) : buffer(4096)
-    {
-        this->archive = archive_read_new();
-        this->source = &source;
+TarArchive::TarArchive(Source& source, bool raw) : buffer(4096)
+{
+    this->archive = archive_read_new();
+    this->source = &source;
 
+    if (!raw) {
         archive_read_support_filter_all(archive);
         archive_read_support_format_all(archive);
-        check(archive_read_open(archive,
-                (void *)this,
-                TarArchive::callback_open,
-                TarArchive::callback_read,
-                TarArchive::callback_close),
-            "failed to open archive: %s");
-    }
-
-    TarArchive(const Path & path)
-    {
-        this->archive = archive_read_new();
-
+    } else {
         archive_read_support_filter_all(archive);
-        archive_read_support_format_all(archive);
-        check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+        archive_read_support_format_raw(archive);
+        archive_read_support_format_empty(archive);
     }
+    check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
+}
 
-    TarArchive(const TarArchive &) = delete;
 
-    void close()
-    {
-        check(archive_read_close(archive), "failed to close archive: %s");
-    }
+TarArchive::TarArchive(const Path &path)
+{
+    this->archive = archive_read_new();
 
-    ~TarArchive()
-    {
-        if (this->archive) archive_read_free(this->archive);
-    }
+    archive_read_support_filter_all(archive);
+    archive_read_support_format_all(archive);
+    check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+}
 
-private:
+void TarArchive::close() {
+    check(archive_read_close(this->archive), "Failed to close archive (%s)");
+}
 
-    static int callback_open(struct archive *, void * self) {
-        return ARCHIVE_OK;
-    }
-
-    static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
-    {
-        auto self = (TarArchive *)_self;
-        *buffer = self->buffer.data();
-
-        try {
-            return self->source->read((char *) self->buffer.data(), 4096);
-        } catch (EndOfFile &) {
-            return 0;
-        } catch (std::exception & err) {
-            archive_set_error(archive, EIO, "source threw exception: %s", err.what());
-            return -1;
-        }
-    }
-
-    static int callback_close(struct archive *, void * self) {
-        return ARCHIVE_OK;
-    }
-};
+TarArchive::~TarArchive() {
+    if (this->archive) archive_read_free(this->archive);
+}
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index 89a024f1d..18adf3490 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -1,7 +1,26 @@
 #include "serialise.hh"
+#include <archive.h>
 
 namespace nix {
 
+struct TarArchive {
+    struct archive *archive;
+    Source *source;
+    std::vector<unsigned char> buffer;
+
+    void check(int err, const char *reason = "Failed to extract archive (%s)");
+
+    TarArchive(Source& source, bool raw = false);
+
+    TarArchive(const Path &path);
+
+    // disable copy constructor
+    TarArchive(const TarArchive&) = delete;
+
+    void close();
+
+    ~TarArchive();
+};
 void unpackTarfile(Source & source, const Path & destDir);
 
 void unpackTarfile(const Path & tarFile, const Path & destDir);

From 0431cf6d0992e7986afbb3d0ffd0a7e1cca8ae8a Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Sun, 7 Feb 2021 15:34:24 -0600
Subject: [PATCH 802/882] fix nixbld user name/uid for macOS

---
 scripts/bigsur-nixbld-user-migration.sh | 46 +++++++++++++++++++++++++
 scripts/install-darwin-multi-user.sh    |  2 ++
 scripts/install-multi-user.sh           |  6 ++--
 3 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100755 scripts/bigsur-nixbld-user-migration.sh

diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh
new file mode 100755
index 000000000..f1619fd56
--- /dev/null
+++ b/scripts/bigsur-nixbld-user-migration.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+((NEW_NIX_FIRST_BUILD_UID=301))
+
+id_available(){
+	dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null
+}
+
+change_nixbld_names_and_ids(){
+	local name uid next_id
+	((next_id=NEW_NIX_FIRST_BUILD_UID))
+	echo "Attempting to migrate nixbld users."
+	echo "Each user should change from nixbld# to _nixbld#"
+	echo "and their IDs relocated to $next_id+"
+	while read -r name uid; do
+		echo "   Checking $name (uid: $uid)"
+		# iterate for a clean ID
+		while id_available "$next_id"; do
+			((next_id++))
+			if ((next_id >= 400)); then
+				echo "We've hit UID 400 without placing all of your users :("
+				echo "You should use the commands in this script as a starting"
+				echo "point to review your UID-space and manually move the"
+				echo "remaining users (or delete them, if you don't need them)."
+				exit 1
+			fi
+		done
+
+		if [[ $name == _* ]]; then
+			echo "      It looks like $name has already been renamed--skipping."
+		else
+			# first 3 are cleanup, it's OK if they aren't here
+			sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true
+			sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true
+			# remove existing user from group
+			sudo dseditgroup -o edit -t user -d $name nixbld || true
+			sudo dscl . change /Users/$name UniqueID $uid $next_id
+			sudo dscl . change /Users/$name RecordName $name _$name
+			# add renamed user to group
+			sudo dseditgroup -o edit -t user -a _$name nixbld
+			echo "      $name migrated to _$name (uid: $next_id)"
+		fi
+	done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2)
+}
+
+change_nixbld_names_and_ids
diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index a27be2a43..f6575ae2f 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -4,6 +4,8 @@ set -eu
 set -o pipefail
 
 readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
+NIX_FIRST_BUILD_UID="301"
+NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 
 dsclattr() {
     /usr/bin/dscl . -read "$1" \
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 5e8b4ac18..30ccf1764 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -25,7 +25,9 @@ readonly RED='\033[31m'
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
-readonly NIX_FIRST_BUILD_UID="30001"
+# darwin installer needs to override these
+NIX_FIRST_BUILD_UID="30001"
+NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
@@ -104,7 +106,7 @@ EOF
 }
 
 nix_user_for_core() {
-    printf "nixbld%d" "$1"
+    printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1"
 }
 
 nix_uid_for_core() {

From 826877cabf9374e0acd5408c6975ee332b1cccc8 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 8 Mar 2021 11:56:33 +0100
Subject: [PATCH 803/882] Add some logic for signing realisations

Not exposed anywhere, but built realisations are now signed (and this
should be forwarded when copy-ing them around)
---
 src/libstore/build/local-derivation-goal.cc | 12 ++++--
 src/libstore/ca-specific-schema.sql         |  1 +
 src/libstore/local-store.cc                 | 29 ++++++++++---
 src/libstore/local-store.hh                 |  4 +-
 src/libstore/realisation.cc                 | 46 ++++++++++++++++++++-
 src/libstore/realisation.hh                 |  8 ++++
 src/libstore/store-api.hh                   |  5 +++
 7 files changed, 93 insertions(+), 12 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 9c2f1dda6..048135ccf 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2615,10 +2615,14 @@ void LocalDerivationGoal::registerOutputs()
        but it's fine to do in all cases. */
 
     if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-        for (auto& [outputName, newInfo] : infos)
-            worker.store.registerDrvOutput(Realisation{
-                .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName},
-                .outPath = newInfo.path});
+        for (auto& [outputName, newInfo] : infos) {
+            auto thisRealisation = Realisation{
+                .id = DrvOutput{initialOutputs.at(outputName).outputHash,
+                                outputName},
+                .outPath = newInfo.path};
+            getLocalStore().signRealisation(thisRealisation);
+            worker.store.registerDrvOutput(thisRealisation);
+        }
     }
 }
 
diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql
index 93c442826..20ee046a1 100644
--- a/src/libstore/ca-specific-schema.sql
+++ b/src/libstore/ca-specific-schema.sql
@@ -6,6 +6,7 @@ create table if not exists Realisations (
     drvPath text not null,
     outputName text not null, -- symbolic output id, usually "out"
     outputPath integer not null,
+    signatures text, -- space-separated list
     primary key (drvPath, outputName),
     foreign key (outputPath) references ValidPaths(id) on delete cascade
 );
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 90fb4a4bd..6bc963f27 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -310,13 +310,13 @@ LocalStore::LocalStore(const Params & params)
     if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
         state->stmts->RegisterRealisedOutput.create(state->db,
             R"(
-                insert or replace into Realisations (drvPath, outputName, outputPath)
-                values (?, ?, (select id from ValidPaths where path = ?))
+                insert or replace into Realisations (drvPath, outputName, outputPath, signatures)
+                values (?, ?, (select id from ValidPaths where path = ?), ?)
                 ;
             )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
-                select Output.path from Realisations
+                select Output.path, Realisations.signatures from Realisations
                     inner join ValidPaths as Output on Output.id = Realisations.outputPath
                     where drvPath = ? and outputName = ?
                     ;
@@ -662,6 +662,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
             (info.id.strHash())
             (info.id.outputName)
             (printStorePath(info.outPath))
+            (concatStringsSep(" ", info.signatures))
             .exec();
     });
 }
@@ -1107,6 +1108,11 @@ bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info)
     return requireSigs && !info.checkSignatures(*this, getPublicKeys());
 }
 
+bool LocalStore::realisationIsUntrusted(const Realisation & realisation)
+{
+    return requireSigs && !realisation.checkSignatures(getPublicKeys());
+}
+
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
@@ -1612,6 +1618,18 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
 }
 
 
+void LocalStore::signRealisation(Realisation & realisation)
+{
+    // FIXME: keep secret keys in memory.
+
+    auto secretKeyFiles = settings.secretKeyFiles;
+
+    for (auto & secretKeyFile : secretKeyFiles.get()) {
+        SecretKey secretKey(readFile(secretKeyFile));
+        realisation.sign(secretKey);
+    }
+}
+
 void LocalStore::signPathInfo(ValidPathInfo & info)
 {
     // FIXME: keep secret keys in memory.
@@ -1649,8 +1667,9 @@ std::optional<const Realisation> LocalStore::queryRealisation(
         if (!use.next())
             return std::nullopt;
         auto outputPath = parseStorePath(use.getStr(0));
-        return Ret{
-            Realisation{.id = id, .outPath = outputPath}};
+        auto signatures = tokenizeString<StringSet>(use.getStr(1));
+        return Ret{Realisation{
+            .id = id, .outPath = outputPath, .signatures = signatures}};
     });
 }
 }  // namespace nix
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index fc67f215a..d54609f01 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -137,6 +137,7 @@ public:
         SubstitutablePathInfos & infos) override;
 
     bool pathInfoIsTrusted(const ValidPathInfo &) override;
+    bool realisationIsUntrusted(const Realisation & ) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
@@ -272,9 +273,10 @@ private:
     bool isValidPath_(State & state, const StorePath & path);
     void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
 
-    /* Add signatures to a ValidPathInfo using the secret keys
+    /* Add signatures to a ValidPathInfo or Realisation using the secret keys
        specified by the ‘secret-key-files’ option. */
     void signPathInfo(ValidPathInfo & info);
+    void signRealisation(Realisation &);
 
     Path getRealStoreDir() override { return realStoreDir; }
 
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index cd74af4ee..638065547 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -25,27 +25,69 @@ nlohmann::json Realisation::toJSON() const {
     return nlohmann::json{
         {"id", id.to_string()},
         {"outPath", outPath.to_string()},
+        {"signatures", signatures},
     };
 }
 
 Realisation Realisation::fromJSON(
     const nlohmann::json& json,
     const std::string& whence) {
-    auto getField = [&](std::string fieldName) -> std::string {
+    auto getOptionalField = [&](std::string fieldName) -> std::optional<std::string> {
         auto fieldIterator = json.find(fieldName);
         if (fieldIterator == json.end())
+            return std::nullopt;
+        return *fieldIterator;
+    };
+    auto getField = [&](std::string fieldName) -> std::string {
+        if (auto field = getOptionalField(fieldName))
+            return *field;
+        else
             throw Error(
                 "Drv output info file '%1%' is corrupt, missing field %2%",
                 whence, fieldName);
-        return *fieldIterator;
     };
 
+    StringSet signatures;
+    if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
+        signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
+
     return Realisation{
         .id = DrvOutput::parse(getField("id")),
         .outPath = StorePath(getField("outPath")),
+        .signatures = signatures,
     };
 }
 
+std::string Realisation::fingerprint() const
+{
+    auto serialized = toJSON();
+    serialized.erase("signatures");
+    return serialized.dump();
+}
+
+void Realisation::sign(const SecretKey & secretKey)
+{
+    signatures.insert(secretKey.signDetached(fingerprint()));
+}
+
+bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
+{
+    return verifyDetached(fingerprint(), sig, publicKeys);
+}
+
+size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
+{
+    // FIXME: Maybe we should return `maxSigs` if the realisation corresponds to
+    // an input-addressed one − because in that case the drv is enough to check
+    // it − but we can't know that here.
+
+    size_t good = 0;
+    for (auto & sig : signatures)
+        if (checkSignature(publicKeys, sig))
+            good++;
+    return good;
+}
+
 StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index fc92d3c17..f5049c9e9 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -3,6 +3,7 @@
 #include "path.hh"
 #include <nlohmann/json_fwd.hpp>
 #include "comparator.hh"
+#include "crypto.hh"
 
 namespace nix {
 
@@ -25,9 +26,16 @@ struct Realisation {
     DrvOutput id;
     StorePath outPath;
 
+    StringSet signatures;
+
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
+    std::string fingerprint() const;
+    void sign(const SecretKey &);
+    bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
+    size_t checkSignatures(const PublicKeys & publicKeys) const;
+
     StorePath getPath() const { return outPath; }
 
     GENERATE_CMP(Realisation, me->id, me->outPath);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 71a28eeb8..0cd56d34e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -389,6 +389,11 @@ public:
         return true;
     }
 
+    virtual bool realisationIsUntrusted(const Realisation & )
+    {
+        return true;
+    }
+
 protected:
 
     virtual void queryPathInfoUncached(const StorePath & path,

From 3e6017f911127555cfbed71fe4a4df8f70d08bbb Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 8 Mar 2021 15:07:33 +0100
Subject: [PATCH 804/882] pathInfoIsTrusted -> pathInfoIsUntrusted

I guess the rationale behind the old name wath that
`pathInfoIsTrusted(info)` returns `true` iff we would need to `blindly`
trust the path (because it has no valid signature and `requireSigs` is
set), but I find it to be a really confusing footgun because it's quite
natural to give it the opposite meaning.
---
 src/libstore/build/substitution-goal.cc | 2 +-
 src/libstore/local-store.cc             | 4 ++--
 src/libstore/local-store.hh             | 2 +-
 src/libstore/store-api.hh               | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 5d88b8758..7b1ac126e 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -142,7 +142,7 @@ void PathSubstitutionGoal::tryNext()
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
-    if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info))
+    if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
     {
         warn("substituter '%s' does not have a valid signature for path '%s'",
             sub->getUri(), worker.store.printStorePath(storePath));
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 6bc963f27..950a9f74e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1103,7 +1103,7 @@ const PublicKeys & LocalStore::getPublicKeys()
     return *state->publicKeys;
 }
 
-bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info)
+bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info)
 {
     return requireSigs && !info.checkSignatures(*this, getPublicKeys());
 }
@@ -1116,7 +1116,7 @@ bool LocalStore::realisationIsUntrusted(const Realisation & realisation)
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    if (checkSigs && pathInfoIsTrusted(info))
+    if (checkSigs && pathInfoIsUntrusted(info))
         throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
 
     addTempRoot(info.path);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index d54609f01..c311d295a 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -136,7 +136,7 @@ public:
     void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
-    bool pathInfoIsTrusted(const ValidPathInfo &) override;
+    bool pathInfoIsUntrusted(const ValidPathInfo &) override;
     bool realisationIsUntrusted(const Realisation & ) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0cd56d34e..b90aeaa4c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -384,7 +384,7 @@ public:
        we don't really want to add the dependencies listed in a nar info we
        don't trust anyyways.
        */
-    virtual bool pathInfoIsTrusted(const ValidPathInfo &)
+    virtual bool pathInfoIsUntrusted(const ValidPathInfo &)
     {
         return true;
     }

From 54ced9072b94515a756e1e8e76c92a42f0ccf366 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 8 Mar 2021 16:43:11 +0100
Subject: [PATCH 805/882] Check the signatures when copying store paths around

Broken atm
---
 src/libstore/local-store.cc |  8 ++++++++
 src/libstore/local-store.hh |  1 +
 src/libstore/store-api.cc   |  2 +-
 src/libstore/store-api.hh   |  2 ++
 tests/ca/signatures.sh      | 39 +++++++++++++++++++++++++++++++++++++
 tests/local.mk              |  3 ++-
 6 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 tests/ca/signatures.sh

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 950a9f74e..83daa7506 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -652,6 +652,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     }
 }
 
+void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
+{
+    settings.requireExperimentalFeature("ca-derivations");
+    if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
+        registerDrvOutput(info);
+    else
+        throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string());
+}
 
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index c311d295a..26e034a82 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -203,6 +203,7 @@ public:
     /* Register the store path 'output' as the output named 'outputName' of
        derivation 'deriver'. */
     void registerDrvOutput(const Realisation & info) override;
+    void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
     void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
 
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 77c310988..5e321cedf 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -798,7 +798,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
     try {
         for (auto & realisation : realisations) {
-            dstStore->registerDrvOutput(realisation);
+            dstStore->registerDrvOutput(realisation, checkSigs);
         }
     } catch (MissingExperimentalFeature & e) {
         // Don't fail if the remote doesn't support CA derivations is it might
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b90aeaa4c..5d19e8949 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -485,6 +485,8 @@ public:
      */
     virtual void registerDrvOutput(const Realisation & output)
     { unsupported("registerDrvOutput"); }
+    virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs)
+    { return registerDrvOutput(output); }
 
     /* Write a NAR dump of a store path. */
     virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
diff --git a/tests/ca/signatures.sh b/tests/ca/signatures.sh
new file mode 100644
index 000000000..4b4e468f7
--- /dev/null
+++ b/tests/ca/signatures.sh
@@ -0,0 +1,39 @@
+source common.sh
+
+# Globally enable the ca derivations experimental flag
+sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf"
+
+clearStore
+clearCache
+
+nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1
+pk1=$(cat $TEST_ROOT/pk1)
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+ensureCorrectlyCopied () {
+    attrPath="$1"
+    nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath"
+}
+
+testOneCopy () {
+    clearStore
+    rm -rf "$REMOTE_STORE_DIR"
+
+    attrPath="$1"
+    nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \
+        --secret-key-files "$TEST_ROOT/sk1"
+
+    ensureCorrectlyCopied "$attrPath"
+
+    # Ensure that we can copy back what we put in the store
+    clearStore
+    nix copy --from $REMOTE_STORE \
+        --file ./content-addressed.nix "$attrPath" \
+        --trusted-public-keys $pk1
+}
+
+for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do
+    testOneCopy "$attrPath"
+done
diff --git a/tests/local.mk b/tests/local.mk
index e17555051..9a227bec5 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -41,8 +41,9 @@ nix_tests = \
   build.sh \
   compute-levels.sh \
   ca/build.sh \
-  ca/nix-copy.sh \
   ca/substitute.sh
+  ca/signatures.sh \
+  ca/nix-copy.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From 703c98c6cb922ff9d8cd8cb2c1104e0d3b15b803 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 8 Mar 2021 17:32:20 +0100
Subject: [PATCH 806/882] Properly sign the unresolved drvs

Don't let them inherit the signature from the parent one (because it
makes no sense to do so), but re-sign them after they have been built
---
 src/libstore/build/derivation-goal.cc       | 2 ++
 src/libstore/build/derivation-goal.hh       | 3 +++
 src/libstore/build/local-derivation-goal.cc | 7 ++++++-
 src/libstore/build/local-derivation-goal.hh | 2 ++
 4 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 7dcd2a6eb..d624e58b9 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -925,6 +925,8 @@ void DerivationGoal::resolvedFinished() {
         if (realisation) {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
+            newRealisation.signatures.clear();
+            signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
         } else {
             // If we don't have a realisation, then it must mean that something
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index c85bcd84f..704b77caf 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -180,6 +180,9 @@ struct DerivationGoal : public Goal
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
+    /* Sign the newly built realisation if the store allows it */
+    virtual void signRealisation(Realisation&) {}
+
     /* Close the log file. */
     void closeLogFile();
 
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 048135ccf..2966bb565 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2620,12 +2620,17 @@ void LocalDerivationGoal::registerOutputs()
                 .id = DrvOutput{initialOutputs.at(outputName).outputHash,
                                 outputName},
                 .outPath = newInfo.path};
-            getLocalStore().signRealisation(thisRealisation);
+            signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }
     }
 }
 
+void LocalDerivationGoal::signRealisation(Realisation & realisation)
+{
+    getLocalStore().signRealisation(realisation);
+}
+
 
 void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 {
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 4bbf27a1b..47b818a8b 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -161,6 +161,8 @@ struct LocalDerivationGoal : public DerivationGoal
        as valid. */
     void registerOutputs() override;
 
+    void signRealisation(Realisation &) override;
+
     /* Check that an output meets the requirements specified by the
        'outputChecks' attribute (or the legacy
        '{allowed,disallowed}{References,Requisites}' attributes). */

From 5869b3025d8ed2b99a8dca61f335789ce6dc83e1 Mon Sep 17 00:00:00 2001
From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org>
Date: Tue, 16 Mar 2021 02:42:14 +0100
Subject: [PATCH 807/882] tests/local.mk: fix missing newline escape

Fixes syntax error introduced in 54ced9072b94515a756e1e8e76c92a42f0ccf366.
---
 tests/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/local.mk b/tests/local.mk
index 1ca363091..de095c117 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -42,7 +42,7 @@ nix_tests = \
   build.sh \
   compute-levels.sh \
   ca/build.sh \
-  ca/substitute.sh
+  ca/substitute.sh \
   ca/signatures.sh \
   ca/nix-copy.sh
   # parallel.sh

From 5716345adf2e794fd62229ea52352e74e92e8e63 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 10 Nov 2020 10:43:33 +0100
Subject: [PATCH 808/882] Add a test ensuring compatibility with an old daemon

This requires adding `nix` to its own closure which is a bit unfortunate,
but as it is optional (the test will be disabled if `OUTER_NIX` is unset) it
shouldn't be too much of an issue.

(Ideally this should go in another derivation so that we can build Nix and run
the test independently, but as the tests are running in the same derivation
as the build it's a bit complicated to do so).
---
 flake.nix                        | 9 +++++++++
 tests/common.sh.in               | 3 +--
 tests/local.mk                   | 2 +-
 tests/remote-store-old-daemon.sh | 7 +++++++
 tests/remote-store.sh            | 4 ++--
 5 files changed, 20 insertions(+), 5 deletions(-)
 create mode 100644 tests/remote-store-old-daemon.sh

diff --git a/flake.nix b/flake.nix
index e59ec9a35..3e236aaca 100644
--- a/flake.nix
+++ b/flake.nix
@@ -150,6 +150,11 @@
       # 'nix.perl-bindings' packages.
       overlay = final: prev: {
 
+        # An older version of Nix to test against when using the daemon.
+        # Currently using `nixUnstable` as the stable one doesn't respect
+        # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
+        mainstream-nix = prev.nixUnstable;
+
         nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
@@ -158,6 +163,8 @@
 
           VERSION_SUFFIX = versionSuffix;
 
+          OUTER_NIX = mainstream-nix;
+
           outputs = [ "out" "dev" "doc" ];
 
           nativeBuildInputs = nativeBuildDeps;
@@ -486,6 +493,8 @@
         stdenv.mkDerivation {
           name = "nix";
 
+          OUTER_NIX = mainstream-nix;
+
           outputs = [ "out" "dev" "doc" ];
 
           nativeBuildInputs = nativeBuildDeps;
diff --git a/tests/common.sh.in b/tests/common.sh.in
index de44a4da4..277dd6dfa 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -57,7 +57,6 @@ clearStore() {
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
     mkdir "$NIX_STATE_DIR"
-    nix-store --init
     clearProfiles
 }
 
@@ -73,7 +72,7 @@ startDaemon() {
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
     rm -f $NIX_STATE_DIR/daemon-socket/socket
-    nix daemon &
+    ${NIX_DAEMON_COMMAND:-nix daemon} &
     for ((i = 0; i < 30; i++)); do
         if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
         sleep 1
diff --git a/tests/local.mk b/tests/local.mk
index de095c117..dd9a0ad56 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -6,7 +6,7 @@ nix_tests = \
   gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
   gc-runtime.sh check-refs.sh filter-source.sh \
-  local-store.sh remote-store.sh export.sh export-graph.sh \
+  local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh \
diff --git a/tests/remote-store-old-daemon.sh b/tests/remote-store-old-daemon.sh
new file mode 100644
index 000000000..ede7ce716
--- /dev/null
+++ b/tests/remote-store-old-daemon.sh
@@ -0,0 +1,7 @@
+# Test that the new Nix can properly talk to an old daemon.
+# If `$OUTER_NIX` isn't set (e.g. when bootsraping), just skip this test
+
+if [[ -n "$OUTER_NIX" ]]; then
+    export NIX_DAEMON_COMMAND=$OUTER_NIX/bin/nix-daemon
+    source remote-store.sh
+fi
diff --git a/tests/remote-store.sh b/tests/remote-store.sh
index f7ae1a2ed..31210ab47 100644
--- a/tests/remote-store.sh
+++ b/tests/remote-store.sh
@@ -23,12 +23,12 @@ startDaemon
 
 storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh
 
+nix-store --gc --max-freed 1K
+
 nix-store --dump-db > $TEST_ROOT/d1
 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2
 cmp $TEST_ROOT/d1 $TEST_ROOT/d2
 
-nix-store --gc --max-freed 1K
-
 killDaemon
 
 user=$(whoami)

From eab9cdbd75e739be33f9433cfba9ab354d084440 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 17 Nov 2020 14:33:09 +0100
Subject: [PATCH 809/882] Add a test for the migration of the db between
 versions

---
 tests/db-migration.sh | 25 +++++++++++++++++++++++++
 tests/local.mk        |  1 +
 2 files changed, 26 insertions(+)
 create mode 100644 tests/db-migration.sh

diff --git a/tests/db-migration.sh b/tests/db-migration.sh
new file mode 100644
index 000000000..e6a405770
--- /dev/null
+++ b/tests/db-migration.sh
@@ -0,0 +1,25 @@
+# Test that we can successfully migrate from an older db schema
+
+# Only run this if we have an older Nix available
+if [[ -z "$OUTER_NIX" ]]; then
+    exit 0
+fi
+
+source common.sh
+
+# Fill the db using the older Nix
+PATH_WITH_NEW_NIX="$PATH"
+export PATH="$OUTER_NIX/bin:$PATH"
+clearStore
+nix-build simple.nix --no-out-link
+nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1
+dependenciesOutPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1")
+fixedOutPath=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build fixed.nix -A good.0 --no-out-link)
+
+# Migrate to the new schema and ensure that everything's there
+export PATH="$PATH_WITH_NEW_NIX"
+info=$(nix path-info --json $dependenciesOutPath)
+[[ $info =~ '"ultimate":true' ]]
+[[ $info =~ 'cache1.example.org' ]]
+nix verify -r "$fixedOutPath"
+nix verify -r "$dependenciesOutPath" --sigs-needed 1 --trusted-public-keys $(cat $TEST_ROOT/pk1)
diff --git a/tests/local.mk b/tests/local.mk
index dd9a0ad56..01c35551f 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -7,6 +7,7 @@ nix_tests = \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
   gc-runtime.sh check-refs.sh filter-source.sh \
   local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \
+  db-migration.sh \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh \

From a0866c8ea4bc66f9aacc7ad19139d57946b3df18 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 16 Mar 2021 13:43:08 +0100
Subject: [PATCH 810/882] Make the tests (optionnally) run in another
 derivation

That way we can run them without rebuilding Nix
---
 flake.nix          | 41 ++++++++++++++++++++++++++++++++++++-----
 tests/common.sh.in |  6 ++++++
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index 3e236aaca..c2e5db53a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -144,6 +144,32 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
+      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
+        NIX_DAEMON_PACKAGE = daemon;
+        NIX_CLIENT_PACKAGE = client;
+        name = "nix-tests-${client.version}-against-${daemon.version}";
+        inherit version;
+
+        src = self;
+
+        VERSION_SUFFIX = versionSuffix;
+
+        nativeBuildInputs = nativeBuildDeps;
+        buildInputs = buildDeps ++ awsDeps;
+        propagatedBuildInputs = propagatedDeps;
+
+        enableParallelBuilding = true;
+
+        dontBuild = true;
+        doInstallCheck = true;
+
+        installPhase = ''
+          mkdir -p $out
+        '';
+        installCheckPhase = "make installcheck";
+
+      };
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
@@ -153,7 +179,7 @@
         # An older version of Nix to test against when using the daemon.
         # Currently using `nixUnstable` as the stable one doesn't respect
         # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
-        mainstream-nix = prev.nixUnstable;
+        nixStable = prev.nix;
 
         nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
           name = "nix-${version}";
@@ -163,8 +189,6 @@
 
           VERSION_SUFFIX = versionSuffix;
 
-          OUTER_NIX = mainstream-nix;
-
           outputs = [ "out" "dev" "doc" ];
 
           nativeBuildInputs = nativeBuildDeps;
@@ -441,6 +465,15 @@
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
+        installTests =
+          let pkgs = nixpkgsFor.${system}; in
+          pkgs.runCommand "install-tests" {
+            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
+            againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
+            # Disabled because the latest stable version doesn't handle
+            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
+          } "touch $out";
       });
 
       packages = forAllSystems (system: {
@@ -493,8 +526,6 @@
         stdenv.mkDerivation {
           name = "nix";
 
-          OUTER_NIX = mainstream-nix;
-
           outputs = [ "out" "dev" "doc" ];
 
           nativeBuildInputs = nativeBuildDeps;
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 277dd6dfa..d31d3fbb8 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -29,6 +29,12 @@ unset XDG_CACHE_HOME
 mkdir -p $TEST_HOME
 
 export PATH=@bindir@:$PATH
+if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then
+  export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH
+fi
+if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
+  export NIX_DAEMON_COMMAND="$NIX_DAEMON_PACKAGE/bin/nix-daemon"
+fi
 coreutils=@coreutils@
 
 export dot=@dot@

From 81df1b5c687b7606f0159485c33bf5f7e2614eba Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 16 Mar 2021 14:15:57 +0100
Subject: [PATCH 811/882] Remove the `remote-store-old-daemon` test

Doesn't make sense anymore with the new setup
---
 tests/local.mk                   | 2 +-
 tests/remote-store-old-daemon.sh | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)
 delete mode 100644 tests/remote-store-old-daemon.sh

diff --git a/tests/local.mk b/tests/local.mk
index 01c35551f..e7e85f97e 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -6,7 +6,7 @@ nix_tests = \
   gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
   gc-runtime.sh check-refs.sh filter-source.sh \
-  local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \
+  local-store.sh remote-store.sh export.sh export-graph.sh \
   db-migration.sh \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
diff --git a/tests/remote-store-old-daemon.sh b/tests/remote-store-old-daemon.sh
deleted file mode 100644
index ede7ce716..000000000
--- a/tests/remote-store-old-daemon.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-# Test that the new Nix can properly talk to an old daemon.
-# If `$OUTER_NIX` isn't set (e.g. when bootsraping), just skip this test
-
-if [[ -n "$OUTER_NIX" ]]; then
-    export NIX_DAEMON_COMMAND=$OUTER_NIX/bin/nix-daemon
-    source remote-store.sh
-fi

From be60c9ef50bf5fa653138802f63727fa0aadf50a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 16 Mar 2021 14:20:10 +0100
Subject: [PATCH 812/882] Fix the db-migration test

---
 tests/db-migration.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/db-migration.sh b/tests/db-migration.sh
index e6a405770..e0ff7d311 100644
--- a/tests/db-migration.sh
+++ b/tests/db-migration.sh
@@ -1,7 +1,8 @@
 # Test that we can successfully migrate from an older db schema
 
 # Only run this if we have an older Nix available
-if [[ -z "$OUTER_NIX" ]]; then
+# XXX: This assumes that the `daemon` package is older than the `client` one
+if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then
     exit 0
 fi
 
@@ -9,7 +10,7 @@ source common.sh
 
 # Fill the db using the older Nix
 PATH_WITH_NEW_NIX="$PATH"
-export PATH="$OUTER_NIX/bin:$PATH"
+export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH"
 clearStore
 nix-build simple.nix --no-out-link
 nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1

From 5ec873b127139ca90cc31967c25c9a34fb4cc3e4 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 16 Mar 2021 16:44:42 +0100
Subject: [PATCH 813/882] Shorten the test drv name

To prevent the OSX build to fail because of a too long socket path
---
 flake.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index c2e5db53a..1cd54e702 100644
--- a/flake.nix
+++ b/flake.nix
@@ -147,7 +147,10 @@
       testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        name = "nix-tests-${client.version}-against-${daemon.version}";
+        # Must keep this name short as OSX has a rather strict limit on the
+        # socket path length, and this name appears in the path of the
+        # nix-daemon socket used in the tests
+        name = "nix-tests";
         inherit version;
 
         src = self;

From 77f5d171e17294ebb017a386d4408bf4613dfed7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Mar 2021 16:53:39 +0100
Subject: [PATCH 814/882] --override-input: Imply --no-write-lock-file

Fixes #3779.
---
 src/libcmd/installables.cc |  3 ++-
 src/nix/flake.cc           | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 7102f5a1a..898e642a5 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -111,10 +111,11 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).",
+        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
         .category = category,
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
+            lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
                 parseFlakeRef(flakeRef, absPath(".")));
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 2f0c468a8..d37791aba 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -110,6 +110,7 @@ struct CmdFlakeUpdate : FlakeCommand
         removeFlag("recreate-lock-file");
         removeFlag("update-input");
         removeFlag("no-update-lock-file");
+        removeFlag("no-write-lock-file");
     }
 
     std::string doc() override
@@ -124,6 +125,7 @@ struct CmdFlakeUpdate : FlakeCommand
         settings.tarballTtl = 0;
 
         lockFlags.recreateLockFile = true;
+        lockFlags.writeLockFile = true;
 
         lockFlake();
     }
@@ -136,6 +138,12 @@ struct CmdFlakeLock : FlakeCommand
         return "create missing lock file entries";
     }
 
+    CmdFlakeLock()
+    {
+        /* Remove flags that don't make sense. */
+        removeFlag("no-write-lock-file");
+    }
+
     std::string doc() override
     {
         return
@@ -147,6 +155,8 @@ struct CmdFlakeLock : FlakeCommand
     {
         settings.tarballTtl = 0;
 
+        lockFlags.writeLockFile = true;
+
         lockFlake();
     }
 };

From 66fa1c7375e4b3073a16df4678cf1d37446ed20b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Mar 2021 17:19:04 +0100
Subject: [PATCH 815/882] Merge 'nix flake {info,list-inputs}' into 'nix flake
 metadata'

Fixes #4613.
---
 src/nix/flake-list-inputs.md                 |  23 ----
 src/nix/{flake-info.md => flake-metadata.md} |  27 ++--
 src/nix/flake.cc                             | 134 ++++++++-----------
 tests/flakes.sh                              |  38 +++---
 4 files changed, 92 insertions(+), 130 deletions(-)
 delete mode 100644 src/nix/flake-list-inputs.md
 rename src/nix/{flake-info.md => flake-metadata.md} (75%)

diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md
deleted file mode 100644
index 250e13be0..000000000
--- a/src/nix/flake-list-inputs.md
+++ /dev/null
@@ -1,23 +0,0 @@
-R""(
-
-# Examples
-
-* Show the inputs of the `hydra` flake:
-
-  ```console
-  # nix flake list-inputs github:NixOS/hydra
-  github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7
-  ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67
-  │   ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f
-  │   └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31
-  └───nixpkgs follows input 'nix/nixpkgs'
-  ```
-
-# Description
-
-This command shows the inputs of the flake specified by the flake
-referenced *flake-url*. Since it prints the locked inputs that result
-from generating or updating the lock file, this command essentially
-displays the contents of the flake's lock file in human-readable form.
-
-)""
diff --git a/src/nix/flake-info.md b/src/nix/flake-metadata.md
similarity index 75%
rename from src/nix/flake-info.md
rename to src/nix/flake-metadata.md
index fda3171db..5a009409b 100644
--- a/src/nix/flake-info.md
+++ b/src/nix/flake-metadata.md
@@ -5,19 +5,24 @@ R""(
 * Show what `nixpkgs` resolves to:
 
   ```console
-  # nix flake info nixpkgs
-  Resolved URL:  github:NixOS/nixpkgs
-  Locked URL:    github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4
-  Description:   A collection of packages for the Nix package manager
-  Path:          /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source
-  Revision:      b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4
-  Last modified: 2020-12-23 12:36:12
+  # nix flake metadata nixpkgs
+  Resolved URL:  github:edolstra/dwarffs
+  Locked URL:    github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5
+  Description:   A filesystem that fetches DWARF debug info from the Internet on demand
+  Path:          /nix/store/769s05vjydmc2lcf6b02az28wsa9ixh1-source
+  Revision:      f691e2c991e75edb22836f1dbe632c40324215c5
+  Last modified: 2021-01-21 15:41:26
+  Inputs:
+  ├───nix: github:NixOS/nix/6254b1f5d298ff73127d7b0f0da48f142bdc753c
+  │   ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f
+  │   └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31
+  └───nixpkgs follows input 'nix/nixpkgs'
   ```
 
 * Show information about `dwarffs` in JSON format:
 
   ```console
-  # nix flake info dwarffs --json | jq .
+  # nix flake metadata dwarffs --json | jq .
   {
     "description": "A filesystem that fetches DWARF debug info from the Internet on demand",
     "lastModified": 1597153508,
@@ -29,6 +34,7 @@ R""(
       "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5",
       "type": "github"
     },
+    "locks": { ... },
     "original": {
       "id": "dwarffs",
       "type": "indirect"
@@ -75,6 +81,9 @@ data. This includes:
   time of the commit of the locked flake; for tarball flakes, it's the
   most recent timestamp of any file inside the tarball.
 
+* `Inputs`: The flake inputs with their corresponding lock file
+  entries.
+
 With `--json`, the output is a JSON object with the following fields:
 
 * `original` and `originalUrl`: The flake reference specified by the
@@ -96,4 +105,6 @@ With `--json`, the output is a JSON object with the following fields:
 
 * `lastModified`: See `Last modified` above.
 
+* `locks`: The contents of `flake.lock`.
+
 )""
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index d37791aba..5ce2e082c 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -43,12 +43,6 @@ public:
         return parseFlakeRef(flakeUrl, absPath(".")); //FIXME
     }
 
-    Flake getFlake()
-    {
-        auto evalState = getEvalState();
-        return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries);
-    }
-
     LockedFlake lockFlake()
     {
         return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags);
@@ -60,43 +54,6 @@ public:
     }
 };
 
-static void printFlakeInfo(const Store & store, const Flake & flake)
-{
-    logger->cout("Resolved URL:  %s", flake.resolvedRef.to_string());
-    logger->cout("Locked URL:    %s", flake.lockedRef.to_string());
-    if (flake.description)
-        logger->cout("Description:   %s", *flake.description);
-    logger->cout("Path:          %s", store.printStorePath(flake.sourceInfo->storePath));
-    if (auto rev = flake.lockedRef.input.getRev())
-        logger->cout("Revision:      %s", rev->to_string(Base16, false));
-    if (auto revCount = flake.lockedRef.input.getRevCount())
-        logger->cout("Revisions:     %s", *revCount);
-    if (auto lastModified = flake.lockedRef.input.getLastModified())
-        logger->cout("Last modified: %s",
-            std::put_time(std::localtime(&*lastModified), "%F %T"));
-}
-
-static nlohmann::json flakeToJSON(const Store & store, const Flake & flake)
-{
-    nlohmann::json j;
-    if (flake.description)
-        j["description"] = *flake.description;
-    j["originalUrl"] = flake.originalRef.to_string();
-    j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs());
-    j["resolvedUrl"] = flake.resolvedRef.to_string();
-    j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs());
-    j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
-    j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs());
-    if (auto rev = flake.lockedRef.input.getRev())
-        j["revision"] = rev->to_string(Base16, false);
-    if (auto revCount = flake.lockedRef.input.getRevCount())
-        j["revCount"] = *revCount;
-    if (auto lastModified = flake.lockedRef.input.getLastModified())
-        j["lastModified"] = *lastModified;
-    j["path"] = store.printStorePath(flake.sourceInfo->storePath);
-    return j;
-}
-
 struct CmdFlakeUpdate : FlakeCommand
 {
     std::string description() override
@@ -175,54 +132,72 @@ static void enumerateOutputs(EvalState & state, Value & vFlake,
         callback(attr.name, *attr.value, *attr.pos);
 }
 
-struct CmdFlakeInfo : FlakeCommand, MixJSON
+struct CmdFlakeMetadata : FlakeCommand, MixJSON
 {
     std::string description() override
     {
-        return "list info about a given flake";
+        return "show flake metadata";
     }
 
     std::string doc() override
     {
         return
-          #include "flake-info.md"
+          #include "flake-metadata.md"
           ;
     }
 
     void run(nix::ref<nix::Store> store) override
     {
-        auto flake = getFlake();
+        auto lockedFlake = lockFlake();
+        auto & flake = lockedFlake.flake;
 
         if (json) {
-            auto json = flakeToJSON(*store, flake);
-            logger->cout("%s", json.dump());
-        } else
-            printFlakeInfo(*store, flake);
-    }
-};
+            nlohmann::json j;
+            if (flake.description)
+                j["description"] = *flake.description;
+            j["originalUrl"] = flake.originalRef.to_string();
+            j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs());
+            j["resolvedUrl"] = flake.resolvedRef.to_string();
+            j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs());
+            j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
+            j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs());
+            if (auto rev = flake.lockedRef.input.getRev())
+                j["revision"] = rev->to_string(Base16, false);
+            if (auto revCount = flake.lockedRef.input.getRevCount())
+                j["revCount"] = *revCount;
+            if (auto lastModified = flake.lockedRef.input.getLastModified())
+                j["lastModified"] = *lastModified;
+            j["path"] = store->printStorePath(flake.sourceInfo->storePath);
+            j["locks"] = lockedFlake.lockFile.toJSON();
+            logger->cout("%s", j.dump());
+        } else {
+            logger->cout(
+                ANSI_BOLD "Resolved URL:" ANSI_NORMAL "  %s",
+                flake.resolvedRef.to_string());
+            logger->cout(
+                ANSI_BOLD "Locked URL:" ANSI_NORMAL "    %s",
+                flake.lockedRef.to_string());
+            if (flake.description)
+                logger->cout(
+                    ANSI_BOLD "Description:" ANSI_NORMAL "   %s",
+                    *flake.description);
+            logger->cout(
+                ANSI_BOLD "Path:" ANSI_NORMAL "          %s",
+                store->printStorePath(flake.sourceInfo->storePath));
+            if (auto rev = flake.lockedRef.input.getRev())
+                logger->cout(
+                    ANSI_BOLD "Revision:" ANSI_NORMAL "      %s",
+                    rev->to_string(Base16, false));
+            if (auto revCount = flake.lockedRef.input.getRevCount())
+                logger->cout(
+                    ANSI_BOLD "Revisions:" ANSI_NORMAL "     %s",
+                    *revCount);
+            if (auto lastModified = flake.lockedRef.input.getLastModified())
+                logger->cout(
+                    ANSI_BOLD "Last modified:" ANSI_NORMAL " %s",
+                    std::put_time(std::localtime(&*lastModified), "%F %T"));
 
-struct CmdFlakeListInputs : FlakeCommand, MixJSON
-{
-    std::string description() override
-    {
-        return "list flake inputs";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "flake-list-inputs.md"
-          ;
-    }
-
-    void run(nix::ref<nix::Store> store) override
-    {
-        auto flake = lockFlake();
-
-        if (json)
-            logger->cout("%s", flake.lockFile.toJSON());
-        else {
-            logger->cout("%s", flake.flake.lockedRef);
+            logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL);
 
             std::unordered_set<std::shared_ptr<Node>> visited;
 
@@ -236,7 +211,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
                     if (auto lockedNode = std::get_if<0>(&input.second)) {
                         logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
                             prefix + (last ? treeLast : treeConn), input.first,
-                            *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef);
+                            *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef);
 
                         bool firstVisit = visited.insert(*lockedNode).second;
 
@@ -249,8 +224,8 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
                 }
             };
 
-            visited.insert(flake.lockFile.root);
-            recurse(*flake.lockFile.root, "");
+            visited.insert(lockedFlake.lockFile.root);
+            recurse(*lockedFlake.lockFile.root, "");
         }
     }
 };
@@ -1048,8 +1023,7 @@ struct CmdFlake : NixMultiCommand
         : MultiCommand({
                 {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
                 {"lock", []() { return make_ref<CmdFlakeLock>(); }},
-                {"info", []() { return make_ref<CmdFlakeInfo>(); }},
-                {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }},
+                {"metadata", []() { return make_ref<CmdFlakeMetadata>(); }},
                 {"check", []() { return make_ref<CmdFlakeCheck>(); }},
                 {"init", []() { return make_ref<CmdFlakeInit>(); }},
                 {"new", []() { return make_ref<CmdFlakeNew>(); }},
diff --git a/tests/flakes.sh b/tests/flakes.sh
index 9747aba7a..e78e4a39d 100644
--- a/tests/flakes.sh
+++ b/tests/flakes.sh
@@ -164,16 +164,17 @@ EOF
 # Test 'nix flake list'.
 [[ $(nix registry list | wc -l) == 7 ]]
 
-# Test 'nix flake info'.
-nix flake info flake1 | grep -q 'URL: .*flake1.*'
+# Test 'nix flake metadata'.
+nix flake metadata flake1
+nix flake metadata flake1 | grep -q 'Locked URL:.*flake1.*'
 
-# Test 'nix flake info' on a local flake.
-(cd $flake1Dir && nix flake info) | grep -q 'URL: .*flake1.*'
-(cd $flake1Dir && nix flake info .) | grep -q 'URL: .*flake1.*'
-nix flake info $flake1Dir | grep -q 'URL: .*flake1.*'
+# Test 'nix flake metadata' on a local flake.
+(cd $flake1Dir && nix flake metadata) | grep -q 'URL:.*flake1.*'
+(cd $flake1Dir && nix flake metadata .) | grep -q 'URL:.*flake1.*'
+nix flake metadata $flake1Dir | grep -q 'URL:.*flake1.*'
 
-# Test 'nix flake info --json'.
-json=$(nix flake info flake1 --json | jq .)
+# Test 'nix flake metadata --json'.
+json=$(nix flake metadata flake1 --json | jq .)
 [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
 [[ -d $(echo "$json" | jq -r .path) ]]
 [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]]
@@ -181,7 +182,7 @@ hash1=$(echo "$json" | jq -r .revision)
 
 echo -n '# foo' >> $flake1Dir/flake.nix
 git -C $flake1Dir commit -a -m 'Foo'
-hash2=$(nix flake info flake1 --json --refresh | jq -r .revision)
+hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision)
 [[ $hash1 != $hash2 ]]
 
 # Test 'nix build' on a flake.
@@ -630,7 +631,7 @@ hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit'
 nix build -o $TEST_ROOT/result hg+file://$flake5Dir
 [[ -e $TEST_ROOT/result/hello ]]
 
-(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision)
+(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
 
 nix eval hg+file://$flake5Dir#expr
 
@@ -638,13 +639,13 @@ nix eval hg+file://$flake5Dir#expr
 
 (! nix eval hg+file://$flake5Dir#expr --no-allow-dirty)
 
-(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision)
+(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
 
 hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file'
 
-nix flake info --json hg+file://$flake5Dir --refresh | jq -e -r .revision
-nix flake info --json hg+file://$flake5Dir
-[[ $(nix flake info --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
+nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision
+nix flake metadata --json hg+file://$flake5Dir
+[[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
 
 nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
 
@@ -654,7 +655,7 @@ tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5
 nix build -o $TEST_ROOT/result file://$TEST_ROOT/flake.tar.gz
 
 # Building with a tarball URL containing a SRI hash should also work.
-url=$(nix flake info --json file://$TEST_ROOT/flake.tar.gz | jq -r .url)
+url=$(nix flake metadata --json file://$TEST_ROOT/flake.tar.gz | jq -r .url)
 [[ $url =~ sha256- ]]
 
 nix build -o $TEST_ROOT/result $url
@@ -680,9 +681,8 @@ nix flake lock $flake3Dir
 nix flake lock $flake3Dir --update-input flake2/flake1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
 
-# Test 'nix flake list-inputs'.
-[[ $(nix flake list-inputs $flake3Dir | wc -l) == 5 ]]
-nix flake list-inputs $flake3Dir --json | jq .
+# Test 'nix flake metadata --json'.
+nix flake metadata $flake3Dir --json | jq .
 
 # Test circular flake dependencies.
 cat > $flakeA/flake.nix <<EOF
@@ -721,4 +721,4 @@ git -C $flakeB commit -a -m 'Foo'
 [[ $(nix eval --update-input b $flakeA#foo) = 1912 ]]
 
 # Test list-inputs with circular dependencies
-nix flake list-inputs $flakeA
+nix flake metadata $flakeA

From ddcef3bcbbba69ab8c43214ca8099f1d3d7e8c08 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 16 Mar 2021 18:51:17 +0100
Subject: [PATCH 816/882] Fix Nix to properly work with stores using a scoped
 IPv6 address

According to RFC4007[1], IPv6 addresses can have a so-called zone_id
separated from the actual address with `%` as delimiter. In contrast to
Nix 2.3, the version on `master` doesn't recognize it as such:

    $ nix ping-store --store ssh://root@fe80::1%18 --experimental-features nix-command
    warning: 'ping-store' is a deprecated alias for 'store ping'
    error: --- Error ----------------------------------------------------------------- nix
    don't know how to open Nix store 'ssh://root@fe80::1%18'

I modified the IPv6 match-regex accordingly to optionally detect this
part of the address. As we don't seem to do anything special with it, I
decided to leave it as part of the URL for now.

Fixes #4490

[1] https://tools.ietf.org/html/rfc4007
---
 src/libutil/tests/url.cc | 18 ++++++++++++++++++
 src/libutil/url-parts.hh |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc
index 80646ad3e..aff58e9ee 100644
--- a/src/libutil/tests/url.cc
+++ b/src/libutil/tests/url.cc
@@ -117,6 +117,24 @@ namespace nix {
         ASSERT_EQ(parsed, expected);
     }
 
+    TEST(parseURL, parseScopedRFC4007IPv6Address) {
+        auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
+            .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
+            .scheme = "http",
+            .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
+            .path = "",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+
+    }
+
     TEST(parseURL, parseIPv6Address) {
         auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
         auto parsed = parseURL(s);
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 862d9fa6e..da10a6bbc 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -8,7 +8,7 @@ namespace nix {
 // URI stuff.
 const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
 const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
-const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+";
+const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?";
 const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
 const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
 const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";

From a61112aadf58c1578cbdcba32b1582d25ca7ed9b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 17 Mar 2021 11:27:11 +0100
Subject: [PATCH 817/882] Remove unimplemented hashAlgoOpt

It was in the header but never implemented.
---
 src/libstore/derivations.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 061d70f69..2df440536 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -52,7 +52,7 @@ struct DerivationOutput
         DerivationOutputCAFloating,
         DerivationOutputDeferred
     > output;
-    std::optional<HashType> hashAlgoOpt(const Store & store) const;
+
     /* Note, when you use this function you should make sure that you're passing
        the right derivation name. When in doubt, you should use the safer
        interface provided by BasicDerivation::outputsAndOptPaths */

From ef83ced4e170130cb6f9acd1d253351b02490658 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Mar 2021 17:21:37 +0100
Subject: [PATCH 818/882] Restore 'nix flake info' as a deprecated alias

---
 src/nix/flake.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 5ce2e082c..a2b6c0303 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -230,6 +230,15 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
     }
 };
 
+struct CmdFlakeInfo : CmdFlakeMetadata
+{
+    void run(nix::ref<nix::Store> store) override
+    {
+        warn("'nix flake info' is a deprecated alias for 'nix flake metadata'");
+        CmdFlakeMetadata::run(store);
+    }
+};
+
 struct CmdFlakeCheck : FlakeCommand
 {
     bool build = true;
@@ -1024,6 +1033,7 @@ struct CmdFlake : NixMultiCommand
                 {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
                 {"lock", []() { return make_ref<CmdFlakeLock>(); }},
                 {"metadata", []() { return make_ref<CmdFlakeMetadata>(); }},
+                {"info", []() { return make_ref<CmdFlakeInfo>(); }},
                 {"check", []() { return make_ref<CmdFlakeCheck>(); }},
                 {"init", []() { return make_ref<CmdFlakeInit>(); }},
                 {"new", []() { return make_ref<CmdFlakeNew>(); }},

From 3e0e443181997c52b0db19ae781948c573a634dd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Mar 2021 17:56:39 +0100
Subject: [PATCH 819/882] ProgressBar: Respect verbosity level

This makes its behaviour consistent with SimpleLogger.
---
 src/libmain/progress-bar.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 0e5432fca..15354549a 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -122,6 +122,7 @@ public:
 
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
+        if (lvl > verbosity) return;
         auto state(state_.lock());
         log(*state, lvl, fs.s);
     }

From 1765711b68c8647b502c2c009dace9632e9300d7 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Wed, 17 Mar 2021 18:43:37 -0400
Subject: [PATCH 820/882] tests/config: Fix config test configuration

First, "XDG_CONFIG_HOME" shouldn't be named "home", as it may be
confusing compared with `$HOME`, which an upcoming test will be using.

Then, using a fixed location for the test is problematic. Use
`$TEST_ROOT` instead.
---
 tests/config.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/config.sh b/tests/config.sh
index eaa46c395..1edc09c1a 100644
--- a/tests/config.sh
+++ b/tests/config.sh
@@ -1,15 +1,15 @@
 source common.sh
 
 # Test that files are loaded from XDG by default
-export XDG_CONFIG_HOME=/tmp/home
-export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2
+export XDG_CONFIG_HOME=$TEST_ROOT/confighome
+export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2
 files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
-[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]]
+[[ $files == "$TEST_ROOT/confighome/nix/nix.conf:$TEST_ROOT/dir1/nix/nix.conf:$TEST_ROOT/dir2/nix/nix.conf" ]]
 
 # Test that setting NIX_USER_CONF_FILES overrides all the default user config files
-export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf
+export NIX_USER_CONF_FILES=$TEST_ROOT/file1.conf:$TEST_ROOT/file2.conf
 files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
-[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]]
+[[ $files == "$TEST_ROOT/file1.conf:$TEST_ROOT/file2.conf" ]]
 
 # Test that it's possible to load the config from a custom location
 here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
@@ -24,4 +24,4 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
 exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
 [[ $prev != $exp_cores ]]
 [[ $exp_cores == "4242" ]]
-[[ $exp_features == "nix-command flakes" ]]
\ No newline at end of file
+[[ $exp_features == "nix-command flakes" ]]

From bf07581497d55ade85d80e5d9ad9bf5d962e3403 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Wed, 17 Mar 2021 19:02:11 -0400
Subject: [PATCH 821/882] tests: Test `.config` stays clean with
 XDG_CONFIG_HOME set

---
 tests/config.sh | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/config.sh b/tests/config.sh
index 1edc09c1a..01c78f2c3 100644
--- a/tests/config.sh
+++ b/tests/config.sh
@@ -1,5 +1,31 @@
 source common.sh
 
+# Isolate the home for this test.
+# Other tests (e.g. flake registry tests) could be writing to $HOME in parallel.
+export HOME=$TEST_ROOT/userhome
+
+# Test that using XDG_CONFIG_HOME works
+# Assert the config folder didn't exist initially.
+[ ! -e "$HOME/.config" ]
+# Without XDG_CONFIG_HOME, creates $HOME/.config
+unset XDG_CONFIG_HOME
+# Run against the nix registry to create the config dir
+# (Tip: this relies on removing non-existent entries being a no-op!)
+nix registry remove userhome-without-xdg
+# Verifies it created it
+[ -e "$HOME/.config" ]
+# Remove the directory it created
+rm -rf "$HOME/.config"
+# Run the same test, but with XDG_CONFIG_HOME
+export XDG_CONFIG_HOME=$TEST_ROOT/confighome
+# Assert the XDG_CONFIG_HOME/nix path does not exist yet.
+[ ! -e "$TEST_ROOT/confighome/nix" ]
+nix registry remove userhome-with-xdg
+# Verifies the confighome path has been created
+[ -e "$TEST_ROOT/confighome/nix" ]
+# Assert the .config folder hasn't been created.
+[ ! -e "$HOME/.config" ]
+
 # Test that files are loaded from XDG by default
 export XDG_CONFIG_HOME=$TEST_ROOT/confighome
 export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2

From 66b857244ff062f6bb97c23e2423338ad242f7a1 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Wed, 17 Mar 2021 17:56:57 -0400
Subject: [PATCH 822/882] Use the appropriate config dir for the registry

---
 src/libfetchers/registry.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 81b2227de..74376adc0 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -114,7 +114,7 @@ static std::shared_ptr<Registry> getSystemRegistry()
 
 Path getUserRegistryPath()
 {
-    return getHome() + "/.config/nix/registry.json";
+    return getConfigDir() + "/nix/registry.json";
 }
 
 std::shared_ptr<Registry> getUserRegistry()

From 9d309de0de9a09d36717abd02a66b51815397d66 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 28 Feb 2021 18:42:46 +0000
Subject: [PATCH 823/882] Clean up serialization for `BuildResult`

A few versioning mistakes were corrected:

- In 27b5747ca7b5599768083dde5fa4d36bfbb0f66f, Daemon protocol had some
  version `>= 0xc` that should have been `>= 0x1c`, or `28` since the
  other conditions used decimal.

- In a2b69660a9b326b95d48bd222993c5225bbd5b5f, legacy SSH gated new CAS
  info on version 6, but version 5 in the server. It is now 6
  everywhere.

Additionally, legacy ssh was sending over more metadata than the daemon
one was. The daemon now sends that data too.

CC @regnat

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libstore/daemon.cc          |  5 ++++-
 src/libstore/remote-store.cc    | 20 ++++++++++++++------
 src/libstore/serve-protocol.hh  |  2 +-
 src/libstore/worker-protocol.hh |  2 +-
 src/nix-store/nix-store.cc      |  2 +-
 5 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index ba7959263..dc9cd2cbd 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -575,7 +575,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
         to << res.status << res.errorMsg;
-        if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) {
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
+            out << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
+        }
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
             worker_proto::write(*store, to, res.builtOutputs);
         }
         break;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 0d884389a..b01cb5a62 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -62,9 +62,15 @@ void write(const Store & store, Sink & out, const Realisation & realisation)
 { out << realisation.toJSON().dump(); }
 
 DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
-{ return DrvOutput::parse(readString(from)); }
+{
+    return DrvOutput::parse(readString(from));
+}
+
 void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
-{ out << drvOutput.to_string(); }
+{
+    out << drvOutput.to_string();
+}
+
 
 std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
 {
@@ -677,10 +683,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     conn->to << buildMode;
     conn.processStderr();
     BuildResult res;
-    unsigned int status;
-    conn->from >> status >> res.errorMsg;
-    res.status = (BuildResult::Status) status;
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) {
+    res.status = (BuildResult::Status) readInt(conn->from);
+    conn->from >> res.errorMsg;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
+        from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
+    }
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
         auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
         res.builtOutputs = builtOutputs;
     }
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index 0a17387cb..02d0810cc 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -5,7 +5,7 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
-#define SERVE_PROTOCOL_VERSION 0x206
+#define SERVE_PROTOCOL_VERSION (2 << 8 | 6)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 95f08bc9a..be071dd78 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x11c
+#define PROTOCOL_VERSION (1 << 8 | 29)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 94d4881dd..b684feccb 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -905,7 +905,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
                     out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
-                if (GET_PROTOCOL_MINOR(clientVersion >= 5)) {
+                if (GET_PROTOCOL_MINOR(clientVersion >= 6)) {
                     worker_proto::write(*store, out, status.builtOutputs);
                 }
 

From f44206e71953501af502354ab1c747aa2412d676 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 22 Mar 2021 15:18:48 +0000
Subject: [PATCH 824/882] Fix typos in the last PR #4656

---
 src/libstore/daemon.cc       | 2 +-
 src/libstore/remote-store.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index dc9cd2cbd..f28ab6438 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -576,7 +576,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->stopWork();
         to << res.status << res.errorMsg;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
-            out << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
+            to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
         }
         if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
             worker_proto::write(*store, to, res.builtOutputs);
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b01cb5a62..ccf095dc2 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -686,7 +686,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     res.status = (BuildResult::Status) readInt(conn->from);
     conn->from >> res.errorMsg;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
-        from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
+        conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
     }
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
         auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});

From 0f40561c78bb5566b42d054620b0576e14fe4627 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Tue, 23 Mar 2021 10:19:00 +0700
Subject: [PATCH 825/882] nix.conf builders: refer to manual page

---
 src/libstore/globals.hh | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index bf0767dfa..3e4ead76c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -207,15 +207,8 @@ public:
     Setting<std::string> builders{
         this, "@" + nixConfDir + "/machines", "builders",
         R"(
-          A semicolon-separated list of build machines, where each machine follows this format:
-
-          {protocol}://{user}@{host} [{comma sep. systems} - {maxJobs} {speedFactor} {comma sep. features}]
-
-          Examples:
-          
-          ssh://root@builder1.com
-          
-          ssh://root@builder2.com x86_64-linux,aarch64-linux - 40 20 nixos-test,benchmark,big-parallel,kvm
+          A semicolon-separated list of build machines.
+          For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md)
         )"};
 
     Setting<bool> buildersUseSubstitutes{

From 71f92741ec979c1059938a638b7fc8da6d7b0936 Mon Sep 17 00:00:00 2001
From: Nicolas Stig124 FORMICHELLA <stigpro@outlook.fr>
Date: Tue, 23 Mar 2021 16:23:24 +0100
Subject: [PATCH 826/882] Added Debian-based OS's profiles

---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 5e8b4ac18..4cc11d210 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -31,7 +31,7 @@ readonly NIX_FIRST_BUILD_UID="30001"
 readonly NIX_ROOT="/nix"
 readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 

From 4638bcfb2cfb74cb5029c0da0af38bb7ca4b4a6f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Mar 2021 16:14:38 +0100
Subject: [PATCH 827/882] Fix some typos

Fixes #4671.
---
 doc/manual/src/command-ref/nix-store.md |  2 +-
 src/libexpr/flake/flake.hh              |  2 +-
 src/libstore/globals.cc                 |  2 +-
 src/libutil/config.cc                   | 28 +++++++++++-----------
 src/libutil/config.hh                   | 22 ++++++++---------
 src/libutil/tests/config.cc             | 32 ++++++++++++-------------
 src/libutil/util.cc                     |  2 +-
 src/nix/build.md                        |  2 +-
 src/nix/flake-init.md                   |  2 +-
 src/nix/flake.md                        |  4 ++--
 src/nix/main.cc                         |  8 +++----
 src/nix/store-prefetch-file.md          |  2 +-
 12 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 361c20cc9..49d06f31e 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -79,7 +79,7 @@ paths. Realisation is a somewhat overloaded term:
     system). If the path is already valid, we are done immediately.
     Otherwise, the path and any missing paths in its closure may be
     produced through substitutes. If there are no (successful)
-    subsitutes, realisation fails.
+    substitutes, realisation fails.
 
 The output path of each derivation is printed on standard output. (For
 non-derivations argument, the argument itself is printed.)
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 65ed1ad0a..d17d5e183 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -113,7 +113,7 @@ struct LockFlags
     /* Whether to commit changes to flake.lock. */
     bool commitLockFile = false;
 
-    /* Flake inputs to be overriden. */
+    /* Flake inputs to be overridden. */
     std::map<InputPath, FlakeRef> inputOverrides;
 
     /* Flake inputs to be updated. This means that any existing lock
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 8d44003f4..d3b27d7be 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -81,7 +81,7 @@ void loadConfFile()
 
     /* We only want to send overrides to the daemon, i.e. stuff from
        ~/.nix/nix.conf or the command line. */
-    globalConfig.resetOverriden();
+    globalConfig.resetOverridden();
 
     auto files = settings.nixUserConfFiles;
     for (auto file = files.rbegin(); file != files.rend(); file++) {
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 7467e5ac0..bda07cd55 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -20,7 +20,7 @@ bool Config::set(const std::string & name, const std::string & value)
             return false;
     }
     i->second.setting->set(value, append);
-    i->second.setting->overriden = true;
+    i->second.setting->overridden = true;
     return true;
 }
 
@@ -35,7 +35,7 @@ void Config::addSetting(AbstractSetting * setting)
     auto i = unknownSettings.find(setting->name);
     if (i != unknownSettings.end()) {
         setting->set(i->second);
-        setting->overriden = true;
+        setting->overridden = true;
         unknownSettings.erase(i);
         set = true;
     }
@@ -48,7 +48,7 @@ void Config::addSetting(AbstractSetting * setting)
                     alias, setting->name);
             else {
                 setting->set(i->second);
-                setting->overriden = true;
+                setting->overridden = true;
                 unknownSettings.erase(i);
                 set = true;
             }
@@ -69,10 +69,10 @@ void AbstractConfig::reapplyUnknownSettings()
         set(s.first, s.second);
 }
 
-void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
+void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (auto & opt : _settings)
-        if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
+        if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
@@ -136,10 +136,10 @@ void AbstractConfig::applyConfigFile(const Path & path)
     } catch (SysError &) { }
 }
 
-void Config::resetOverriden()
+void Config::resetOverridden()
 {
     for (auto & s : _settings)
-        s.second.setting->overriden = false;
+        s.second.setting->overridden = false;
 }
 
 nlohmann::json Config::toJSON()
@@ -169,7 +169,7 @@ AbstractSetting::AbstractSetting(
 
 void AbstractSetting::setDefault(const std::string & str)
 {
-    if (!overriden) set(str);
+    if (!overridden) set(str);
 }
 
 nlohmann::json AbstractSetting::toJSON()
@@ -203,7 +203,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
         .description = fmt("Set the `%s` setting.", name),
         .category = category,
         .labels = {"value"},
-        .handler = {[=](std::string s) { overriden = true; set(s); }},
+        .handler = {[=](std::string s) { overridden = true; set(s); }},
     });
 
     if (isAppendable())
@@ -212,7 +212,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
             .description = fmt("Append to the `%s` setting.", name),
             .category = category,
             .labels = {"value"},
-            .handler = {[=](std::string s) { overriden = true; set(s, true); }},
+            .handler = {[=](std::string s) { overridden = true; set(s, true); }},
         });
 }
 
@@ -365,16 +365,16 @@ bool GlobalConfig::set(const std::string & name, const std::string & value)
     return false;
 }
 
-void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
+void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (auto & config : *configRegistrations)
-        config->getSettings(res, overridenOnly);
+        config->getSettings(res, overriddenOnly);
 }
 
-void GlobalConfig::resetOverriden()
+void GlobalConfig::resetOverridden()
 {
     for (auto & config : *configRegistrations)
-        config->resetOverriden();
+        config->resetOverridden();
 }
 
 nlohmann::json GlobalConfig::toJSON()
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 71e31656d..bf81b4892 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -71,9 +71,9 @@ public:
     /**
      * Adds the currently known settings to the given result map `res`.
      * - res: map to store settings in
-     * - overridenOnly: when set to true only overridden settings will be added to `res`
+     * - overriddenOnly: when set to true only overridden settings will be added to `res`
      */
-    virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
+    virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) = 0;
 
     /**
      * Parses the configuration in `contents` and applies it
@@ -91,7 +91,7 @@ public:
     /**
      * Resets the `overridden` flag of all Settings
      */
-    virtual void resetOverriden() = 0;
+    virtual void resetOverridden() = 0;
 
     /**
      * Outputs all settings to JSON
@@ -127,7 +127,7 @@ public:
 
      MyClass() : Config(readConfigFile("/etc/my-app.conf"))
      {
-       std::cout << foo << "\n"; // will print 123 unless overriden
+       std::cout << foo << "\n"; // will print 123 unless overridden
      }
    };
 */
@@ -163,9 +163,9 @@ public:
 
     void addSetting(AbstractSetting * setting);
 
-    void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
+    void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) override;
 
-    void resetOverriden() override;
+    void resetOverridden() override;
 
     nlohmann::json toJSON() override;
 
@@ -184,7 +184,7 @@ public:
 
     int created = 123;
 
-    bool overriden = false;
+    bool overridden = false;
 
     void setDefault(const std::string & str);
 
@@ -215,7 +215,7 @@ protected:
 
     virtual void convertToArg(Args & args, const std::string & category);
 
-    bool isOverriden() const { return overriden; }
+    bool isOverridden() const { return overridden; }
 };
 
 /* A setting of type T. */
@@ -252,7 +252,7 @@ public:
 
     virtual void override(const T & v)
     {
-        overriden = true;
+        overridden = true;
         value = v;
     }
 
@@ -324,9 +324,9 @@ struct GlobalConfig : public AbstractConfig
 
     bool set(const std::string & name, const std::string & value) override;
 
-    void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
+    void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) override;
 
-    void resetOverriden() override;
+    void resetOverridden() override;
 
     nlohmann::json toJSON() override;
 
diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index c305af9f5..0ebdaf3db 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -29,20 +29,20 @@ namespace nix {
         std::map<std::string, Config::SettingInfo> settings;
         Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
 
-        config.getSettings(settings, /* overridenOnly = */ false);
+        config.getSettings(settings, /* overriddenOnly = */ false);
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "");
         ASSERT_EQ(iter->second.description, "description\n");
     }
 
-    TEST(Config, getDefinedOverridenSettingNotSet) {
+    TEST(Config, getDefinedOverriddenSettingNotSet) {
         Config config;
         std::string value;
         std::map<std::string, Config::SettingInfo> settings;
         Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
 
-        config.getSettings(settings, /* overridenOnly = */ true);
+        config.getSettings(settings, /* overriddenOnly = */ true);
         const auto e = settings.find("name-of-the-setting");
         ASSERT_EQ(e, settings.end());
     }
@@ -55,7 +55,7 @@ namespace nix {
 
         setting.assign("value");
 
-        config.getSettings(settings, /* overridenOnly = */ false);
+        config.getSettings(settings, /* overriddenOnly = */ false);
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "value");
@@ -69,7 +69,7 @@ namespace nix {
 
         ASSERT_TRUE(config.set("name-of-the-setting", "value"));
 
-        config.getSettings(settings, /* overridenOnly = */ false);
+        config.getSettings(settings, /* overriddenOnly = */ false);
         const auto e = settings.find("name-of-the-setting");
         ASSERT_NE(e, settings.end());
         ASSERT_EQ(e->second.value, "value");
@@ -100,7 +100,7 @@ namespace nix {
 
         {
             std::map<std::string, Config::SettingInfo> settings;
-            config.getSettings(settings, /* overridenOnly = */ false);
+            config.getSettings(settings, /* overriddenOnly = */ false);
             ASSERT_EQ(settings.find("key"), settings.end());
         }
 
@@ -108,17 +108,17 @@ namespace nix {
 
         {
             std::map<std::string, Config::SettingInfo> settings;
-            config.getSettings(settings, /* overridenOnly = */ false);
+            config.getSettings(settings, /* overriddenOnly = */ false);
             ASSERT_EQ(settings["key"].value, "value");
         }
     }
 
-    TEST(Config, resetOverriden) {
+    TEST(Config, resetOverridden) {
         Config config;
-        config.resetOverriden();
+        config.resetOverridden();
     }
 
-    TEST(Config, resetOverridenWithSetting) {
+    TEST(Config, resetOverriddenWithSetting) {
         Config config;
         Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
 
@@ -127,7 +127,7 @@ namespace nix {
 
             setting.set("foo");
             ASSERT_EQ(setting.get(), "foo");
-            config.getSettings(settings, /* overridenOnly = */ true);
+            config.getSettings(settings, /* overriddenOnly = */ true);
             ASSERT_TRUE(settings.empty());
         }
 
@@ -135,18 +135,18 @@ namespace nix {
             std::map<std::string, Config::SettingInfo> settings;
 
             setting.override("bar");
-            ASSERT_TRUE(setting.overriden);
+            ASSERT_TRUE(setting.overridden);
             ASSERT_EQ(setting.get(), "bar");
-            config.getSettings(settings, /* overridenOnly = */ true);
+            config.getSettings(settings, /* overriddenOnly = */ true);
             ASSERT_FALSE(settings.empty());
         }
 
         {
             std::map<std::string, Config::SettingInfo> settings;
 
-            config.resetOverriden();
-            ASSERT_FALSE(setting.overriden);
-            config.getSettings(settings, /* overridenOnly = */ true);
+            config.resetOverridden();
+            ASSERT_FALSE(setting.overridden);
+            config.getSettings(settings, /* overriddenOnly = */ true);
             ASSERT_TRUE(settings.empty());
         }
     }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index ef37275ac..dea9c74b7 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1590,7 +1590,7 @@ void startSignalHandlerThread()
     updateWindowSize();
 
     if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
-        throw SysError("quering signal mask");
+        throw SysError("querying signal mask");
 
     sigset_t set;
     sigemptyset(&set);
diff --git a/src/nix/build.md b/src/nix/build.md
index c2f3e387a..20138b7e0 100644
--- a/src/nix/build.md
+++ b/src/nix/build.md
@@ -81,7 +81,7 @@ path installables are substituted.
 
 Unless `--no-link` is specified, after a successful build, it creates
 symlinks to the store paths of the installables. These symlinks have
-the prefix `./result` by default; this can be overriden using the
+the prefix `./result` by default; this can be overridden using the
 `--out-link` option. Each symlink has a suffix `-<N>-<outname>`, where
 *N* is the index of the installable (with the left-most installable
 having index 0), and *outname* is the symbolic derivation output name
diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md
index c66154ad5..890038016 100644
--- a/src/nix/flake-init.md
+++ b/src/nix/flake-init.md
@@ -24,7 +24,7 @@ R""(
 
 This command creates a flake in the current directory by copying the
 files of a template. It will not overwrite existing files. The default
-template is `templates#defaultTemplate`, but this can be overriden
+template is `templates#defaultTemplate`, but this can be overridden
 using `-t`.
 
 # Template definitions
diff --git a/src/nix/flake.md b/src/nix/flake.md
index 440c45dd1..0035195e5 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -70,7 +70,7 @@ Here are some examples of flake references in their URL-like representation:
 * `/home/alice/src/patchelf`: A flake in some other directory.
 * `nixpkgs`: The `nixpkgs` entry in the flake registry.
 * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs`
-  entry in the flake registry, with its Git revision overriden to a
+  entry in the flake registry, with its Git revision overridden to a
   specific value.
 * `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs`
   repository on GitHub.
@@ -377,7 +377,7 @@ outputs = { self, nixpkgs, grcov }: {
 };
 ```
 
-Transitive inputs can be overriden from a `flake.nix` file. For
+Transitive inputs can be overridden from a `flake.nix` file. For
 example, the following overrides the `nixpkgs` input of the `nixops`
 input:
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 06e221682..f8701ee56 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -309,13 +309,13 @@ void mainWrapped(int argc, char * * argv)
 
     if (!args.useNet) {
         // FIXME: should check for command line overrides only.
-        if (!settings.useSubstitutes.overriden)
+        if (!settings.useSubstitutes.overridden)
             settings.useSubstitutes = false;
-        if (!settings.tarballTtl.overriden)
+        if (!settings.tarballTtl.overridden)
             settings.tarballTtl = std::numeric_limits<unsigned int>::max();
-        if (!fileTransferSettings.tries.overriden)
+        if (!fileTransferSettings.tries.overridden)
             fileTransferSettings.tries = 0;
-        if (!fileTransferSettings.connectTimeout.overriden)
+        if (!fileTransferSettings.connectTimeout.overridden)
             fileTransferSettings.connectTimeout = 1;
     }
 
diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md
index 1663b847b..f9fdcbc57 100644
--- a/src/nix/store-prefetch-file.md
+++ b/src/nix/store-prefetch-file.md
@@ -27,6 +27,6 @@ the resulting store path and the cryptographic hash of the contents of
 the file.
 
 The name component of the store path defaults to the last component of
-*url*, but this can be overriden using `--name`.
+*url*, but this can be overridden using `--name`.
 
 )""

From dd77f71afe6733e9790dd001125c423cb648b7ce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Mar 2021 17:10:15 +0100
Subject: [PATCH 828/882] LocalBinaryCacheStore::upsertFile(): Fix race

When multiple threads try to upsert the same file, this could fail.

Fixes #4667.
---
 src/libstore/local-binary-cache-store.cc | 5 ++++-
 tests/ca/substitute.sh                   | 3 +++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index a58b7733f..964c4017e 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -2,6 +2,8 @@
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
 
+#include <atomic>
+
 namespace nix {
 
 struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
@@ -50,7 +52,8 @@ protected:
         const std::string & mimeType) override
     {
         auto path2 = binaryCacheDir + "/" + path;
-        Path tmp = path2 + ".tmp." + std::to_string(getpid());
+        static std::atomic<int> counter{0};
+        Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);
         AutoDelete del(tmp, false);
         StreamToSourceAdapter source(istream);
         writeFile(tmp, source);
diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
index 79a6ef8b1..b44fe499a 100644
--- a/tests/ca/substitute.sh
+++ b/tests/ca/substitute.sh
@@ -6,6 +6,8 @@ source common.sh
 
 sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
 
+rm -rf $TEST_ROOT/binary_cache
+
 export REMOTE_STORE=file://$TEST_ROOT/binary_cache
 
 buildDrvs () {
@@ -13,6 +15,7 @@ buildDrvs () {
 }
 
 # Populate the remote cache
+clearStore
 buildDrvs --post-build-hook ../push-to-store.sh
 
 # Restart the build on an empty store, ensuring that we don't build

From ce791535f63502215d3d41b6ca8d9e62c5fb72e9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 Mar 2021 14:54:05 +0200
Subject: [PATCH 829/882] nixpkgs/master compatibility

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index e59ec9a35..925017472 100644
--- a/flake.nix
+++ b/flake.nix
@@ -90,7 +90,7 @@
             lowdown
             gmock
           ]
-          ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
+          ++ lib.optionals stdenv.isLinux [libseccomp (pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional stdenv.isx86_64 libcpuid;
 

From edd606ae62e213c2a30ff76b8eea4f75ea703d41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Sat, 27 Mar 2021 14:15:28 +0100
Subject: [PATCH 830/882] fetchGit: don't prefix "refs/heads/" on ref = "HEAD"

This fixes builtins.fetchGit { url = ...; ref = "HEAD"; }, that works in
stable nix (v2.3.10), but is broken in nix master:

  $ ./result/bin/nix repl
  Welcome to Nix version 2.4pre19700101_dd77f71. Type :? for help.

  nix-repl> builtins.fetchGit { url = "https://github.com/NixOS/nix"; ref = "HEAD"; }
  fetching Git repository 'https://github.com/NixOS/nix'fatal: couldn't find remote ref refs/heads/HEAD
  error: program 'git' failed with exit code 128

The documentation for builtins.fetchGit says ref = "HEAD" is the
default, so it should also be supported to explicitly pass it.

I came across this issue because poetry2nix can use ref = "HEAD" in some
situations.

Fixes #4674.
---
 src/libfetchers/git.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 4f9db1bcd..b9a240b13 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -365,7 +365,9 @@ struct GitInputScheme : InputScheme
                         ? "refs/*"
                         : ref->compare(0, 5, "refs/") == 0
                             ? *ref
-                            : "refs/heads/" + *ref;
+                            : ref == "HEAD"
+                                ? *ref
+                                : "refs/heads/" + *ref;
                     runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;

From f2a799b16d193a651f682da3ad2103c20ac82d48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Tue, 30 Mar 2021 11:39:37 +0200
Subject: [PATCH 831/882] tests: check that builtins.fetchGit { ..., ref =
 "HEAD"; } works

---
 tests/fetchGit.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 1e8963d76..88744ee7f 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -179,3 +179,13 @@ git clone --depth 1 file://$repo $TEST_ROOT/shallow
 path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
 [[ $path3 = $path6 ]]
 [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
+
+# Explicit ref = "HEAD" should work, and produce the same outPath as without ref
+path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath")
+path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath")
+[[ $path7 = $path8 ]]
+
+# ref = "HEAD" should fetch the HEAD revision
+rev4=$(git -C $repo rev-parse HEAD)
+rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev")
+[[ $rev4 = $rev4_nix ]]

From f3f228700a52857fe6e8632df4e935551ea219ff Mon Sep 17 00:00:00 2001
From: Mykola Orliuk <virkony@gmail.com>
Date: Wed, 31 Mar 2021 04:20:41 +0200
Subject: [PATCH 832/882] canonPath in one pass

---
 src/libutil/util.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index dea9c74b7..c092076f3 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -143,16 +143,18 @@ Path canonPath(const Path & path, bool resolveSymlinks)
             s += '/';
             while (i != end && *i != '/') s += *i++;
 
-            /* If s points to a symlink, resolve it and restart (since
-               the symlink target might contain new symlinks). */
+            /* If s points to a symlink, resolve it and continue from there */
             if (resolveSymlinks && isLink(s)) {
                 if (++followCount >= maxFollow)
                     throw Error("infinite symlink recursion in path '%1%'", path);
-                temp = absPath(readLink(s), dirOf(s))
-                    + string(i, end);
-                i = temp.begin(); /* restart */
+                temp = readLink(s) + string(i, end);
+                i = temp.begin();
                 end = temp.end();
-                s = "";
+                if (!temp.empty() && temp[0] == '/') {
+                    s.clear();  /* restart for symlinks pointing to absolute path */
+                } else {
+                    s = dirOf(s);
+                }
             }
         }
     }

From f66fb5fb5b1478a5da39d0e9cc0f835272199c5d Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 23 Mar 2021 12:06:43 +0100
Subject: [PATCH 833/882] flake.nix: Build nix with strictDeps = true

---
 flake.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/flake.nix b/flake.nix
index 58dc5019d..adb796a05 100644
--- a/flake.nix
+++ b/flake.nix
@@ -233,6 +233,8 @@
 
           separateDebugInfo = true;
 
+          strictDeps = true;
+
           passthru.perl-bindings = with final; stdenv.mkDerivation {
             name = "nix-perl-${version}";
 
@@ -517,6 +519,8 @@
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           stripAllList = ["bin"];
+
+          strictDeps = true;
         };
       });
 

From c3090bc6fdf6e052cd4c56fce6aeb11ddeb5dd6f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 24 Mar 2021 14:44:20 +0100
Subject: [PATCH 834/882] tests/*: show when tests are skipped

---
 tests/build-remote.sh  | 4 ++--
 tests/gc-runtime.sh    | 2 +-
 tests/linux-sandbox.sh | 4 ++--
 tests/recursive.sh     | 2 +-
 tests/shell.sh         | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 04848e4b5..70f82e939 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -1,5 +1,5 @@
-if ! canUseSandbox; then exit; fi
-if ! [[ $busybox =~ busybox ]]; then exit; fi
+if ! canUseSandbox; then exit 99; fi
+if ! [[ $busybox =~ busybox ]]; then exit 99; fi
 
 unset NIX_STORE_DIR
 unset NIX_STATE_DIR
diff --git a/tests/gc-runtime.sh b/tests/gc-runtime.sh
index 4c5028005..6094959cb 100644
--- a/tests/gc-runtime.sh
+++ b/tests/gc-runtime.sh
@@ -4,7 +4,7 @@ case $system in
     *linux*)
         ;;
     *)
-        exit 0;
+        exit 99;
 esac
 
 set -m # enable job control, needed for kill
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 70a90a907..eac62d461 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -2,13 +2,13 @@ source common.sh
 
 clearStore
 
-if ! canUseSandbox; then exit; fi
+if ! canUseSandbox; then exit 99; fi
 
 # Note: we need to bind-mount $SHELL into the chroot. Currently we
 # only support the case where $SHELL is in the Nix store, because
 # otherwise things get complicated (e.g. if it's in /bin, do we need
 # /lib as well?).
-if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
+if [[ ! $SHELL =~ /nix/store ]]; then exit 99; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0
diff --git a/tests/recursive.sh b/tests/recursive.sh
index b020ec710..a55b061b5 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -1,7 +1,7 @@
 source common.sh
 
 # FIXME
-if [[ $(uname) != Linux ]]; then exit; fi
+if [[ $(uname) != Linux ]]; then exit 99; fi
 
 clearStore
 
diff --git a/tests/shell.sh b/tests/shell.sh
index 7a9ee8ab0..2b85bb337 100644
--- a/tests/shell.sh
+++ b/tests/shell.sh
@@ -6,7 +6,7 @@ clearCache
 nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
 nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
 
-if ! canUseSandbox; then exit; fi
+if ! canUseSandbox; then exit 99; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0

From ff1a2143aa1338ccba0e2bc5ccd66bd3df8baa31 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 24 Mar 2021 14:50:15 +0100
Subject: [PATCH 835/882] flake.nix: Make the sandbox tests work again

---
 flake.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index adb796a05..9a758eafa 100644
--- a/flake.nix
+++ b/flake.nix
@@ -78,7 +78,8 @@
             buildPackages.git
             buildPackages.mercurial
             buildPackages.jq
-          ];
+          ]
+          ++ lib.optionals stdenv.isLinux [(pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)];
 
         buildDeps =
           [ curl
@@ -90,7 +91,7 @@
             lowdown
             gmock
           ]
-          ++ lib.optionals stdenv.isLinux [libseccomp (pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)]
+          ++ lib.optionals stdenv.isLinux [libseccomp]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional stdenv.isx86_64 libcpuid;
 

From 5926200db09ca4d0c5769edf24a3cf2e9f472d23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gohla?=
 <51823984+cgohla@users.noreply.github.com>
Date: Thu, 1 Apr 2021 22:54:09 +0100
Subject: [PATCH 836/882] [prerequisites]: add JSON lib dependency

---
 doc/manual/src/installation/prerequisites-source.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index 6825af707..12758c5e1 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -69,3 +69,6 @@
     `--disable-seccomp-sandboxing` option to the `configure` script (Not
     recommended unless your system doesn't support `libseccomp`). To get
     the library, visit <https://github.com/seccomp/libseccomp>.
+ 
+  - Niels Lohmann's [JSON library](https://github.com/nlohmann/json).
+  

From 00f00a995458776e33fdda692abe2099196ac566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 2 Apr 2021 21:32:09 +0200
Subject: [PATCH 837/882] bump actions

---
 .github/workflows/test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2531a7d35..33035ca1e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,9 +13,9 @@ jobs:
     - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v12
+    - uses: cachix/install-nix-action@v13
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v8
+    - uses: cachix/cachix-action@v9
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -44,8 +44,8 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v12
-    - uses: cachix/cachix-action@v8
+    - uses: cachix/install-nix-action@v13
+    - uses: cachix/cachix-action@v9
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -62,7 +62,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2.3.4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@master
+    - uses: cachix/install-nix-action@v13
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"

From f7d9f7c3381acef38e4db2bb2f9e0287c289be54 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 1 Mar 2021 05:48:01 +0000
Subject: [PATCH 838/882] Pull out Buildable into its own file/header in
 libnixstore

---
 src/libcmd/installables.cc | 25 -------------------------
 src/libcmd/installables.hh | 22 +---------------------
 src/libstore/buildable.cc  | 33 +++++++++++++++++++++++++++++++++
 src/libstore/buildable.hh  | 34 ++++++++++++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 46 deletions(-)
 create mode 100644 src/libstore/buildable.cc
 create mode 100644 src/libstore/buildable.hh

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 898e642a5..ca416b9ee 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -20,31 +20,6 @@
 
 namespace nix {
 
-nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["path"] = store->printStorePath(path);
-    return res;
-}
-
-nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["drvPath"] = store->printStorePath(drvPath);
-    for (const auto& [output, path] : outputs) {
-        res["outputs"][output] = path ? store->printStorePath(*path) : "";
-    }
-    return res;
-}
-
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
-    auto res = nlohmann::json::array();
-    for (const Buildable & buildable : buildables) {
-        std::visit([&res, store](const auto & buildable) {
-            res.push_back(buildable.toJSON(store));
-        }, buildable);
-    }
-    return res;
-}
-
 void completeFlakeInputPath(
     ref<EvalState> evalState,
     const FlakeRef & flakeRef,
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index b714f097b..d31afd3d5 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -2,13 +2,12 @@
 
 #include "util.hh"
 #include "path.hh"
+#include "buildable.hh"
 #include "eval.hh"
 #include "flake/flake.hh"
 
 #include <optional>
 
-#include <nlohmann/json_fwd.hpp>
-
 namespace nix {
 
 struct DrvInfo;
@@ -16,25 +15,6 @@ struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }
 
-struct BuildableOpaque {
-    StorePath path;
-    nlohmann::json toJSON(ref<Store> store) const;
-};
-
-struct BuildableFromDrv {
-    StorePath drvPath;
-    std::map<std::string, std::optional<StorePath>> outputs;
-    nlohmann::json toJSON(ref<Store> store) const;
-};
-
-typedef std::variant<
-    BuildableOpaque,
-    BuildableFromDrv
-> Buildable;
-
-typedef std::vector<Buildable> Buildables;
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
-
 struct App
 {
     std::vector<StorePathWithOutputs> context;
diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc
new file mode 100644
index 000000000..5cba45b1d
--- /dev/null
+++ b/src/libstore/buildable.cc
@@ -0,0 +1,33 @@
+#include "buildable.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
+    nlohmann::json res;
+    res["path"] = store->printStorePath(path);
+    return res;
+}
+
+nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
+    nlohmann::json res;
+    res["drvPath"] = store->printStorePath(drvPath);
+    for (const auto& [output, path] : outputs) {
+        res["outputs"][output] = path ? store->printStorePath(*path) : "";
+    }
+    return res;
+}
+
+nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
+    auto res = nlohmann::json::array();
+    for (const Buildable & buildable : buildables) {
+        std::visit([&res, store](const auto & buildable) {
+            res.push_back(buildable.toJSON(store));
+        }, buildable);
+    }
+    return res;
+}
+
+}
diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh
new file mode 100644
index 000000000..6177237be
--- /dev/null
+++ b/src/libstore/buildable.hh
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "util.hh"
+#include "path.hh"
+
+#include <optional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+class Store;
+
+struct BuildableOpaque {
+    StorePath path;
+    nlohmann::json toJSON(ref<Store> store) const;
+};
+
+struct BuildableFromDrv {
+    StorePath drvPath;
+    std::map<std::string, std::optional<StorePath>> outputs;
+    nlohmann::json toJSON(ref<Store> store) const;
+};
+
+typedef std::variant<
+    BuildableOpaque,
+    BuildableFromDrv
+> Buildable;
+
+typedef std::vector<Buildable> Buildables;
+
+nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
+
+}

From 7a2b566dc8f0f94fdd6acbce90e47cd967f9f134 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 2 Mar 2021 00:47:00 +0000
Subject: [PATCH 839/882] Move `StorePathWithOutputs` into its own header/file

In the following commits it will become less prevalent.
---
 src/libcmd/installables.hh        |  1 +
 src/libstore/derivations.cc       |  8 -------
 src/libstore/path-with-outputs.cc | 36 +++++++++++++++++++++++++++++++
 src/libstore/path-with-outputs.hh | 17 +++++++++++++++
 src/libstore/path.cc              | 15 -------------
 src/libstore/path.hh              | 10 ---------
 src/libstore/store-api.cc         |  7 ------
 src/libstore/store-api.hh         |  1 +
 8 files changed, 55 insertions(+), 40 deletions(-)
 create mode 100644 src/libstore/path-with-outputs.cc
 create mode 100644 src/libstore/path-with-outputs.hh

diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index d31afd3d5..e5c6fe208 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -2,6 +2,7 @@
 
 #include "util.hh"
 #include "path.hh"
+#include "path-with-outputs.hh"
 #include "buildable.hh"
 #include "eval.hh"
 #include "flake/flake.hh"
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index fe98182bb..f6defd98f 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -590,14 +590,6 @@ std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& d
 }
 
 
-std::string StorePathWithOutputs::to_string(const Store & store) const
-{
-    return outputs.empty()
-        ? store.printStorePath(path)
-        : store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
-}
-
-
 bool wantOutput(const string & output, const std::set<string> & wanted)
 {
     return wanted.empty() || wanted.find(output) != wanted.end();
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
new file mode 100644
index 000000000..ba15df0a9
--- /dev/null
+++ b/src/libstore/path-with-outputs.cc
@@ -0,0 +1,36 @@
+#include "store-api.hh"
+
+namespace nix {
+
+std::string StorePathWithOutputs::to_string(const Store & store) const
+{
+    return outputs.empty()
+        ? store.printStorePath(path)
+        : store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
+}
+
+
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
+{
+    size_t n = s.find("!");
+    return n == s.npos
+        ? std::make_pair(s, std::set<string>())
+        : std::make_pair(((std::string_view) s).substr(0, n),
+            tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
+}
+
+
+StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
+{
+    auto [path, outputs] = nix::parsePathWithOutputs(s);
+    return {parseStorePath(path), std::move(outputs)};
+}
+
+
+StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
+{
+    auto [path2, outputs] = nix::parsePathWithOutputs(path);
+    return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
+}
+
+}
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
new file mode 100644
index 000000000..a9e3fc7c2
--- /dev/null
+++ b/src/libstore/path-with-outputs.hh
@@ -0,0 +1,17 @@
+#pragma once
+
+#include "path.hh"
+
+namespace nix {
+
+struct StorePathWithOutputs
+{
+    StorePath path;
+    std::set<std::string> outputs;
+
+    std::string to_string(const Store & store) const;
+};
+
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
+
+}
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index dc9dc3897..e642abcd5 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const
     return res;
 }
 
-std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
-{
-    size_t n = s.find("!");
-    return n == s.npos
-        ? std::make_pair(s, std::set<string>())
-        : std::make_pair(((std::string_view) s).substr(0, n),
-            tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
-}
-
-StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
-{
-    auto [path, outputs] = nix::parsePathWithOutputs(s);
-    return {parseStorePath(path), std::move(outputs)};
-}
-
 }
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index b03a0f69d..06ba0663b 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -69,16 +69,6 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
-struct StorePathWithOutputs
-{
-    StorePath path;
-    std::set<std::string> outputs;
-
-    std::string to_string(const Store & store) const;
-};
-
-std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
-
 }
 
 namespace std {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 5e321cedf..e3500872c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -53,13 +53,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
 }
 
 
-StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
-{
-    auto [path2, outputs] = nix::parsePathWithOutputs(path);
-    return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
-}
-
-
 /* Store paths have the following form:
 
    <realized-path> = <store>/<h>-<name>
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5d19e8949..7adbe3b17 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -2,6 +2,7 @@
 
 #include "realisation.hh"
 #include "path.hh"
+#include "path-with-outputs.hh"
 #include "hash.hh"
 #include "content-address.hh"
 #include "serialise.hh"

From 32f4454b9fa3ac30d58e738ece322eb19a0728ba Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 2 Mar 2021 01:06:08 +0000
Subject: [PATCH 840/882] Move `StorePathWithOutput` utilities out of store
 class

These are by no means part of the notion of a store, but rather are
things that happen to use stores. (Or put another way, there's no way
we'd make them virtual methods any time soon.) It's better to move them
out of that too-big class then.

Also, this helps us remove StorePathWithOutputs from the Store interface
altogether next commit.
---
 src/libexpr/get-drvs.cc           |  2 +-
 src/libstore/daemon.cc            |  4 ++--
 src/libstore/path-with-outputs.cc | 12 ++++++------
 src/libstore/path-with-outputs.hh |  9 +++++++++
 src/libstore/store-api.hh         |  7 -------
 src/nix-store/nix-store.cc        |  4 ++--
 6 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 1a3990ea1..7793f26ff 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
-    auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
 
     this->drvPath = store->printStorePath(drvPath);
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index f28ab6438..48706bff8 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopBuildPaths: {
         std::vector<StorePathWithOutputs> drvs;
         for (auto & s : readStrings<Strings>(from))
-            drvs.push_back(store->parsePathWithOutputs(s));
+            drvs.push_back(parsePathWithOutputs(*store, s));
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -861,7 +861,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryMissing: {
         std::vector<StorePathWithOutputs> targets;
         for (auto & s : readStrings<Strings>(from))
-            targets.push_back(store->parsePathWithOutputs(s));
+            targets.push_back(parsePathWithOutputs(*store, s));
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         uint64_t downloadSize, narSize;
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
index ba15df0a9..a898ad09c 100644
--- a/src/libstore/path-with-outputs.cc
+++ b/src/libstore/path-with-outputs.cc
@@ -20,17 +20,17 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
 }
 
 
-StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
+StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
 {
-    auto [path, outputs] = nix::parsePathWithOutputs(s);
-    return {parseStorePath(path), std::move(outputs)};
+    auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
+    return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };
 }
 
 
-StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
+StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
 {
-    auto [path2, outputs] = nix::parsePathWithOutputs(path);
-    return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
+    auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
+    return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
 }
 
 }
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
index a9e3fc7c2..0e34b5aa1 100644
--- a/src/libstore/path-with-outputs.hh
+++ b/src/libstore/path-with-outputs.hh
@@ -14,4 +14,13 @@ struct StorePathWithOutputs
 
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
 
+class Store;
+
+/* Split a string specifying a derivation and a set of outputs
+   (/nix/store/hash-foo!out1,out2,...) into the derivation path
+   and the outputs. */
+StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+
+StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 7adbe3b17..da7ac4460 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -262,11 +262,6 @@ public:
 
     PathSet printStorePathSet(const StorePathSet & path) const;
 
-    /* Split a string specifying a derivation and a set of outputs
-       (/nix/store/hash-foo!out1,out2,...) into the derivation path
-       and the outputs. */
-    StorePathWithOutputs parsePathWithOutputs(const string & s);
-
     /* Display a set of paths in human-readable form (i.e., between quotes
        and separated by commas). */
     std::string showPaths(const StorePathSet & paths);
@@ -290,8 +285,6 @@ public:
        result. */
     StorePath followLinksToStorePath(std::string_view path) const;
 
-    StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
-
     /* Constructs a unique store path name. */
     StorePath makeStorePath(std::string_view type,
         std::string_view hash, std::string_view name) const;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index b684feccb..bfd1299fc 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
 
     std::vector<StorePathWithOutputs> paths;
     for (auto & i : opArgs)
-        paths.push_back(store->followLinksToStorePathWithOutputs(i));
+        paths.push_back(followLinksToStorePathWithOutputs(*store, i));
 
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
@@ -873,7 +873,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 std::vector<StorePathWithOutputs> paths;
                 for (auto & s : readStrings<Strings>(in))
-                    paths.push_back(store->parsePathWithOutputs(s));
+                    paths.push_back(parsePathWithOutputs(*store, s));
 
                 getBuildSettings();
 

From 255d145ba7ac907d1cba8d088da556b591627756 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 2 Mar 2021 03:50:41 +0000
Subject: [PATCH 841/882] Use `BuildableReq` for `buildPaths` and `ensurePath`

This avoids an ambiguity where the `StorePathWithOutputs { drvPath, {}
}` could mean "build `brvPath`" or "substitute `drvPath`" depending on
context.

It also brings the internals closer in line to the new CLI, by
generalizing the `Buildable` type is used there and makes that
distinction already.

In doing so, relegate `StorePathWithOutputs` to being a type just for
backwards compatibility (CLI and RPC).
---
 src/libcmd/installables.cc                  |  7 +--
 src/libexpr/get-drvs.cc                     |  1 +
 src/libexpr/primops.cc                      | 12 +++--
 src/libmain/shared.cc                       |  2 +-
 src/libmain/shared.hh                       |  3 +-
 src/libstore/build/derivation-goal.cc       |  4 +-
 src/libstore/build/entry-points.cc          | 16 +++---
 src/libstore/build/local-derivation-goal.cc | 52 +++++++++++++------
 src/libstore/build/local-derivation-goal.hh |  1 +
 src/libstore/build/worker.cc                |  6 +--
 src/libstore/buildable.cc                   | 47 +++++++++++++++++
 src/libstore/buildable.hh                   | 29 +++++++++--
 src/libstore/daemon.cc                      | 21 +++++---
 src/libstore/legacy-ssh-store.cc            | 16 ++++--
 src/libstore/misc.cc                        | 49 +++++++++---------
 src/libstore/path-with-outputs.cc           | 35 +++++++++++++
 src/libstore/path-with-outputs.hh           |  9 ++++
 src/libstore/remote-store.cc                | 57 +++++++++++++++++----
 src/libstore/remote-store.hh                |  4 +-
 src/libstore/store-api.cc                   |  8 +--
 src/libstore/store-api.hh                   |  6 +--
 src/libstore/worker-protocol.hh             | 22 ++++++++
 src/nix-build/nix-build.cc                  |  4 +-
 src/nix-env/nix-env.cc                      | 30 ++++++-----
 src/nix-env/user-env.cc                     |  9 +++-
 src/nix-store/nix-store.cc                  | 11 ++--
 src/nix/bundle.cc                           |  4 +-
 src/nix/develop.cc                          |  3 +-
 src/nix/flake.cc                            |  5 +-
 src/nix/profile.cc                          | 15 +++---
 src/nix/run.cc                              |  2 +-
 31 files changed, 364 insertions(+), 126 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index ca416b9ee..b68c5f6a7 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -679,19 +679,20 @@ Buildables build(ref<Store> store, Realise mode,
 
     Buildables buildables;
 
-    std::vector<StorePathWithOutputs> pathsToBuild;
+    std::vector<BuildableReq> pathsToBuild;
 
     for (auto & i : installables) {
         for (auto & b : i->toBuildables()) {
             std::visit(overloaded {
                 [&](BuildableOpaque bo) {
-                    pathsToBuild.push_back({bo.path});
+                    pathsToBuild.push_back(bo);
                 },
                 [&](BuildableFromDrv bfd) {
                     StringSet outputNames;
                     for (auto & output : bfd.outputs)
                         outputNames.insert(output.first);
-                    pathsToBuild.push_back({bfd.drvPath, outputNames});
+                    pathsToBuild.push_back(
+                        BuildableReqFromDrv{bfd.drvPath, outputNames});
                 },
             }, b);
             buildables.push_back(std::move(b));
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 7793f26ff..f774e6493 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
+#include "path-with-outputs.hh"
 
 #include <cstring>
 #include <regex>
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 1d1afa768..24bc34b74 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) :
 
 void EvalState::realiseContext(const PathSet & context)
 {
-    std::vector<StorePathWithOutputs> drvs;
+    std::vector<BuildableReqFromDrv> drvs;
 
     for (auto & i : context) {
         auto [ctxS, outputName] = decodeContext(i);
@@ -43,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context)
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
         if (!outputName.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
+            drvs.push_back({ctx, {outputName}});
         }
     }
 
@@ -51,14 +51,16 @@ void EvalState::realiseContext(const PathSet & context)
 
     if (!evalSettings.enableImportFromDerivation)
         throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
-            store->printStorePath(drvs.begin()->path));
+            store->printStorePath(drvs.begin()->drvPath));
 
     /* For performance, prefetch all substitute info. */
     StorePathSet willBuild, willSubstitute, unknown;
     uint64_t downloadSize, narSize;
-    store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+    std::vector<BuildableReq> buildReqs;
+    for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d });
+    store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
-    store->buildPaths(drvs);
+    store->buildPaths(buildReqs);
 
     /* Add the output of this derivations to the allowed
        paths. */
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 5baaff3e9..20027e099 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -36,7 +36,7 @@ void printGCWarning()
 }
 
 
-void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
+void printMissing(ref<Store> store, const std::vector<BuildableReq> & paths, Verbosity lvl)
 {
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index edc7b5efa..18e0fb57d 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -4,6 +4,7 @@
 #include "args.hh"
 #include "common-args.hh"
 #include "path.hh"
+#include "buildable.hh"
 
 #include <signal.h>
 
@@ -42,7 +43,7 @@ struct StorePathWithOutputs;
 
 void printMissing(
     ref<Store> store,
-    const std::vector<StorePathWithOutputs> & paths,
+    const std::vector<BuildableReq> & paths,
     Verbosity lvl = lvlInfo);
 
 void printMissing(ref<Store> store, const StorePathSet & willBuild,
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 2e7be517e..8680d0bce 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
-        StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
+        to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs }));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
     state = &DerivationGoal::haveDerivation;
     name = fmt(
         "building of '%s' from in-memory derivation",
-        StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
+        to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() }));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 686364440..d1973d78b 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -6,16 +6,20 @@
 
 namespace nix {
 
-void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+void Store::buildPaths(const std::vector<BuildableReq> & reqs, BuildMode buildMode)
 {
     Worker worker(*this);
 
     Goals goals;
-    for (auto & path : drvPaths) {
-        if (path.path.isDerivation())
-            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
-        else
-            goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
+    for (auto & br : reqs) {
+        std::visit(overloaded {
+            [&](BuildableReqFromDrv bfd) {
+                goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
+            },
+            [&](BuildableOpaque bo) {
+                goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
+            },
+        }, br);
     }
 
     worker.run(goals);
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 8ef43c225..c245527c9 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1190,6 +1190,26 @@ void LocalDerivationGoal::writeStructuredAttrs()
     chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
+
+static StorePath pathPartOfReq(const BuildableReq & req)
+{
+    return std::visit(overloaded {
+        [&](BuildableOpaque bo) {
+            return bo.path;
+        },
+        [&](BuildableReqFromDrv bfd)  {
+            return bfd.drvPath;
+        },
+    }, req);
+}
+
+
+bool LocalDerivationGoal::isAllowed(const BuildableReq & req)
+{
+    return this->isAllowed(pathPartOfReq(req));
+}
+
+
 struct RestrictedStoreConfig : virtual LocalFSStoreConfig
 {
     using LocalFSStoreConfig::LocalFSStoreConfig;
@@ -1312,25 +1332,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     // an allowed derivation
     { throw Error("queryRealisation"); }
 
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
+    void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
         StorePathSet newPaths;
 
-        for (auto & path : paths) {
-            if (!goal.isAllowed(path.path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
+        for (auto & req : paths) {
+            if (!goal.isAllowed(req))
+                throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req));
         }
 
         next->buildPaths(paths, buildMode);
 
         for (auto & path : paths) {
-            if (!path.path.isDerivation()) continue;
-            auto outputs = next->queryDerivationOutputMap(path.path);
-            for (auto & output : outputs)
-                if (wantOutput(output.first, path.outputs))
-                    newPaths.insert(output.second);
+            auto p =  std::get_if<BuildableReqFromDrv>(&path);
+            if (!p) continue;
+            auto & bfd = *p;
+            auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
+            for (auto & [outputName, outputPath] : outputs)
+                if (wantOutput(outputName, bfd.outputs))
+                    newPaths.insert(outputPath);
         }
 
         StorePathSet closure;
@@ -1358,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override
     { unsupported("addSignatures"); }
 
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+    void queryMissing(const std::vector<BuildableReq> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override
     {
@@ -1366,12 +1388,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
            client about what paths will be built/substituted or are
            already present. Probably not a big deal. */
 
-        std::vector<StorePathWithOutputs> allowed;
-        for (auto & path : targets) {
-            if (goal.isAllowed(path.path))
-                allowed.emplace_back(path);
+        std::vector<BuildableReq> allowed;
+        for (auto & req : targets) {
+            if (goal.isAllowed(req))
+                allowed.emplace_back(req);
             else
-                unknown.insert(path.path);
+                unknown.insert(pathPartOfReq(req));
         }
 
         next->queryMissing(allowed, willBuild, willSubstitute,
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 47b818a8b..edb93f84e 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -116,6 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
+    bool isAllowed(const BuildableReq & req);
 
     friend struct RestrictedStore;
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 616b17e61..fef4cb0cb 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal)
 
 void Worker::run(const Goals & _topGoals)
 {
-    std::vector<nix::StorePathWithOutputs> topPaths;
+    std::vector<nix::BuildableReq> topPaths;
 
     for (auto & i : _topGoals) {
         topGoals.insert(i);
         if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
-            topPaths.push_back({goal->drvPath, goal->wantedOutputs});
+            topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs});
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
-            topPaths.push_back({goal->storePath});
+            topPaths.push_back(BuildableOpaque{goal->storePath});
         }
     }
 
diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc
index 5cba45b1d..63ca1779e 100644
--- a/src/libstore/buildable.cc
+++ b/src/libstore/buildable.cc
@@ -11,6 +11,7 @@ nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
     return res;
 }
 
+template<>
 nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
     nlohmann::json res;
     res["drvPath"] = store->printStorePath(drvPath);
@@ -30,4 +31,50 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store)
     return res;
 }
 
+
+std::string BuildableOpaque::to_string(const Store & store) const {
+    return store.printStorePath(path);
+}
+
+template<>
+std::string BuildableReqFromDrv::to_string(const Store & store) const {
+    return store.printStorePath(drvPath)
+        + "!"
+        + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
+}
+
+std::string to_string(const Store & store, const BuildableReq & req)
+{
+    return std::visit(
+        [&](const auto & req) { return req.to_string(store); },
+        req);
+}
+
+
+BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s)
+{
+    return {store.parseStorePath(s)};
+}
+
+template<>
+BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s)
+{
+    size_t n = s.find("!");
+    assert(n != s.npos);
+    auto drvPath = store.parseStorePath(s.substr(0, n));
+    auto outputsS = s.substr(n + 1);
+    std::set<string> outputs;
+    if (outputsS != "*")
+        outputs = tokenizeString<std::set<string>>(outputsS);
+    return {drvPath, outputs};
+}
+
+BuildableReq parseBuildableReq(const Store & store, std::string_view s)
+{
+    size_t n = s.find("!");
+    return n == s.npos
+        ? (BuildableReq) BuildableOpaque::parse(store, s)
+        : (BuildableReq) BuildableReqFromDrv::parse(store, s);
+}
+
 }
diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh
index 6177237be..db78316bd 100644
--- a/src/libstore/buildable.hh
+++ b/src/libstore/buildable.hh
@@ -2,6 +2,7 @@
 
 #include "util.hh"
 #include "path.hh"
+#include "path.hh"
 
 #include <optional>
 
@@ -13,19 +14,37 @@ class Store;
 
 struct BuildableOpaque {
     StorePath path;
+
     nlohmann::json toJSON(ref<Store> store) const;
+    std::string to_string(const Store & store) const;
+    static BuildableOpaque parse(const Store & store, std::string_view);
 };
 
-struct BuildableFromDrv {
+template<typename Outputs>
+struct BuildableForFromDrv {
     StorePath drvPath;
-    std::map<std::string, std::optional<StorePath>> outputs;
+    Outputs outputs;
+
     nlohmann::json toJSON(ref<Store> store) const;
+    std::string to_string(const Store & store) const;
+    static BuildableForFromDrv<Outputs> parse(const Store & store, std::string_view);
 };
 
-typedef std::variant<
+template <typename Outputs>
+using BuildableFor = std::variant<
     BuildableOpaque,
-    BuildableFromDrv
-> Buildable;
+    BuildableForFromDrv<Outputs>
+>;
+
+typedef BuildableForFromDrv<std::set<std::string>> BuildableReqFromDrv;
+typedef BuildableFor<std::set<std::string>> BuildableReq;
+
+std::string to_string(const Store & store, const BuildableReq &);
+
+BuildableReq parseBuildableReq(const Store & store, std::string_view);
+
+typedef BuildableForFromDrv<std::map<std::string, std::optional<StorePath>>> BuildableFromDrv;
+typedef BuildableFor<std::map<std::string, std::optional<StorePath>>> Buildable;
 
 typedef std::vector<Buildable> Buildables;
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 48706bff8..6b527dcb2 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -2,6 +2,7 @@
 #include "monitor-fd.hh"
 #include "worker-protocol.hh"
 #include "store-api.hh"
+#include "path-with-outputs.hh"
 #include "finally.hh"
 #include "affinity.hh"
 #include "archive.hh"
@@ -259,6 +260,18 @@ static void writeValidPathInfo(
     }
 }
 
+static std::vector<BuildableReq> readBuildableReqs(Store & store, unsigned int clientVersion, Source & from)
+{
+    std::vector<BuildableReq> reqs;
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
+        reqs = worker_proto::read(store, from, Phantom<std::vector<BuildableReq>> {});
+    } else {
+        for (auto & s : readStrings<Strings>(from))
+            reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq());
+    }
+    return reqs;
+}
+
 static void performOp(TunnelLogger * logger, ref<Store> store,
     TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
     Source & from, BufferedSink & to, unsigned int op)
@@ -493,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopBuildPaths: {
-        std::vector<StorePathWithOutputs> drvs;
-        for (auto & s : readStrings<Strings>(from))
-            drvs.push_back(parsePathWithOutputs(*store, s));
+        auto drvs = readBuildableReqs(*store, clientVersion, from);
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -859,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryMissing: {
-        std::vector<StorePathWithOutputs> targets;
-        for (auto & s : readStrings<Strings>(from))
-            targets.push_back(parsePathWithOutputs(*store, s));
+        auto targets = readBuildableReqs(*store, clientVersion, from);
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         uint64_t downloadSize, narSize;
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index a9f53bad9..1cb977be6 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -3,6 +3,7 @@
 #include "remote-store.hh"
 #include "serve-protocol.hh"
 #include "store-api.hh"
+#include "path-with-outputs.hh"
 #include "worker-protocol.hh"
 #include "ssh.hh"
 #include "derivations.hh"
@@ -266,14 +267,23 @@ public:
         return status;
     }
 
-    void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override
+    void buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode) override
     {
         auto conn(connections->get());
 
         conn->to << cmdBuildPaths;
         Strings ss;
-        for (auto & p : drvPaths)
-            ss.push_back(p.to_string(*this));
+        for (auto & p : drvPaths) {
+            auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
+            std::visit(overloaded {
+                [&](StorePathWithOutputs s) {
+                    ss.push_back(s.to_string(*this));
+                },
+                [&](StorePath drvPath) {
+                    throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
+                },
+            }, sOrDrvPath);
+        }
         conn->to << ss;
 
         putBuildSettings(*conn);
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index f58816ad8..e702a4f9e 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -117,7 +117,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
     return std::nullopt;
 }
 
-void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+void Store::queryMissing(const std::vector<BuildableReq> & targets,
     StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
     uint64_t & downloadSize_, uint64_t & narSize_)
 {
@@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
 
     Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
 
-    std::function<void(StorePathWithOutputs)> doPath;
+    std::function<void(BuildableReq)> doPath;
 
     auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
         {
@@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         }
 
         for (auto & i : drv.inputDrvs)
-            pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second }));
+            pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second }));
     };
 
     auto checkOutput = [&](
@@ -177,24 +177,25 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 drvState->outPaths.insert(outPath);
                 if (!drvState->left) {
                     for (auto & path : drvState->outPaths)
-                        pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } ));
+                        pool.enqueue(std::bind(doPath, BuildableOpaque { path } ));
                 }
             }
         }
     };
 
-    doPath = [&](const StorePathWithOutputs & path) {
+    doPath = [&](const BuildableReq & req) {
 
         {
             auto state(state_.lock());
-            if (!state->done.insert(path.to_string(*this)).second) return;
+            if (!state->done.insert(to_string(*this, req)).second) return;
         }
 
-        if (path.path.isDerivation()) {
-            if (!isValidPath(path.path)) {
+        std::visit(overloaded {
+          [&](BuildableReqFromDrv bfd) {
+            if (!isValidPath(bfd.drvPath)) {
                 // FIXME: we could try to substitute the derivation.
                 auto state(state_.lock());
-                state->unknown.insert(path.path);
+                state->unknown.insert(bfd.drvPath);
                 return;
             }
 
@@ -202,52 +203,54 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             /* true for regular derivations, and CA derivations for which we
                have a trust mapping for all wanted outputs. */
             auto knownOutputPaths = true;
-            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
+            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) {
                 if (!pathOpt) {
                     knownOutputPaths = false;
                     break;
                 }
-                if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
+                if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt))
                     invalid.insert(*pathOpt);
             }
             if (knownOutputPaths && invalid.empty()) return;
 
-            auto drv = make_ref<Derivation>(derivationFromPath(path.path));
-            ParsedDerivation parsedDrv(StorePath(path.path), *drv);
+            auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
+            ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
 
             if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
-                    pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState));
+                    pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState));
             } else
-                mustBuildDrv(path.path, *drv);
+                mustBuildDrv(bfd.drvPath, *drv);
 
-        } else {
+          },
+          [&](BuildableOpaque bo) {
 
-            if (isValidPath(path.path)) return;
+            if (isValidPath(bo.path)) return;
 
             SubstitutablePathInfos infos;
-            querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
+            querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos);
 
             if (infos.empty()) {
                 auto state(state_.lock());
-                state->unknown.insert(path.path);
+                state->unknown.insert(bo.path);
                 return;
             }
 
-            auto info = infos.find(path.path);
+            auto info = infos.find(bo.path);
             assert(info != infos.end());
 
             {
                 auto state(state_.lock());
-                state->willSubstitute.insert(path.path);
+                state->willSubstitute.insert(bo.path);
                 state->downloadSize += info->second.downloadSize;
                 state->narSize += info->second.narSize;
             }
 
             for (auto & ref : info->second.references)
-                pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref }));
-        }
+                pool.enqueue(std::bind(doPath, BuildableOpaque { ref }));
+          },
+        }, req);
     };
 
     for (auto & path : targets)
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
index a898ad09c..353286ac6 100644
--- a/src/libstore/path-with-outputs.cc
+++ b/src/libstore/path-with-outputs.cc
@@ -1,3 +1,4 @@
+#include "path-with-outputs.hh"
 #include "store-api.hh"
 
 namespace nix {
@@ -10,6 +11,40 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
 }
 
 
+BuildableReq StorePathWithOutputs::toBuildableReq() const
+{
+    if (!outputs.empty() || path.isDerivation())
+        return BuildableReqFromDrv { path, outputs };
+    else
+        return BuildableOpaque { path };
+}
+
+
+std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs> ss)
+{
+	std::vector<BuildableReq> reqs;
+	for (auto & s : ss) reqs.push_back(s.toBuildableReq());
+	return reqs;
+}
+
+
+std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p)
+{
+    return std::visit(overloaded {
+        [&](BuildableOpaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
+            if (bo.path.isDerivation()) {
+                // drv path gets interpreted as "build", not "get drv file itself"
+                return bo.path;
+            }
+            return StorePathWithOutputs { bo.path };
+        },
+        [&](BuildableReqFromDrv bfd) -> std::variant<StorePathWithOutputs, StorePath> {
+            return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
+        },
+    }, p);
+}
+
+
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
 {
     size_t n = s.find("!");
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
index 0e34b5aa1..870cac08e 100644
--- a/src/libstore/path-with-outputs.hh
+++ b/src/libstore/path-with-outputs.hh
@@ -1,6 +1,9 @@
 #pragma once
 
+#include <variant>
+
 #include "path.hh"
+#include "buildable.hh"
 
 namespace nix {
 
@@ -10,8 +13,14 @@ struct StorePathWithOutputs
     std::set<std::string> outputs;
 
     std::string to_string(const Store & store) const;
+
+    BuildableReq toBuildableReq() const;
+
+    static std::variant<StorePathWithOutputs, StorePath> tryFromBuildableReq(const BuildableReq &);
 };
 
+std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs>);
+
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
 
 class Store;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index ccf095dc2..de1c95ed6 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -1,5 +1,6 @@
 #include "serialise.hh"
 #include "util.hh"
+#include "path-with-outputs.hh"
 #include "remote-fs-accessor.hh"
 #include "remote-store.hh"
 #include "worker-protocol.hh"
@@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
     out << renderContentAddress(ca);
 }
 
+
+BuildableReq read(const Store & store, Source & from, Phantom<BuildableReq> _)
+{
+    auto s = readString(from);
+    return parseBuildableReq(store, s);
+}
+
+void write(const Store & store, Sink & out, const BuildableReq & req)
+{
+    out << to_string(store, req);
+}
+
+
 Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
 {
     std::string rawInput = readString(from);
@@ -58,8 +72,12 @@ Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
         "remote-protocol"
     );
 }
+
 void write(const Store & store, Sink & out, const Realisation & realisation)
-{ out << realisation.toJSON().dump(); }
+{
+    out << realisation.toJSON().dump();
+}
+
 
 DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
 {
@@ -652,16 +670,36 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
     return {Realisation{.id = id, .outPath = *outPaths.begin()}};
 }
 
+static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector<BuildableReq> & reqs)
+{
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
+        worker_proto::write(store, conn->to, reqs);
+    } else {
+        Strings ss;
+        for (auto & p : reqs) {
+            auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
+            std::visit(overloaded {
+                [&](StorePathWithOutputs s) {
+                    ss.push_back(s.to_string(store));
+                },
+                [&](StorePath drvPath) {
+                    throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
+                        store.printStorePath(drvPath),
+                        GET_PROTOCOL_MAJOR(conn->daemonVersion),
+                        GET_PROTOCOL_MINOR(conn->daemonVersion));
+                },
+            }, sOrDrvPath);
+        }
+        conn->to << ss;
+    }
+}
 
-void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+void RemoteStore::buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode)
 {
     auto conn(getConnection());
     conn->to << wopBuildPaths;
     assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
-    Strings ss;
-    for (auto & p : drvPaths)
-        ss.push_back(p.to_string(*this));
-    conn->to << ss;
+    writeBuildableReqs(*this, conn, drvPaths);
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
         conn->to << buildMode;
     else
@@ -800,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
 }
 
 
-void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+void RemoteStore::queryMissing(const std::vector<BuildableReq> & targets,
     StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
     uint64_t & downloadSize, uint64_t & narSize)
 {
@@ -811,10 +849,7 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
             // to prevent a deadlock.
             goto fallback;
         conn->to << wopQueryMissing;
-        Strings ss;
-        for (auto & p : targets)
-            ss.push_back(p.to_string(*this));
-        conn->to << ss;
+        writeBuildableReqs(*this, conn, targets);
         conn.processStderr();
         willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
         willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index b3a9910a3..20d366038 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -85,7 +85,7 @@ public:
 
     std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
 
-    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
+    void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override;
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override;
@@ -108,7 +108,7 @@ public:
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
-    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+    void queryMissing(const std::vector<BuildableReq> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e3500872c..8b60bdc62 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath,
 
 void Store::substitutePaths(const StorePathSet & paths)
 {
-    std::vector<StorePathWithOutputs> paths2;
+    std::vector<BuildableReq> paths2;
     for (auto & path : paths)
         if (!path.isDerivation())
-            paths2.push_back({path});
+            paths2.push_back(BuildableOpaque{path});
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
     queryMissing(paths2,
@@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths)
 
     if (!willSubstitute.empty())
         try {
-            std::vector<StorePathWithOutputs> subs;
-            for (auto & p : willSubstitute) subs.push_back({p});
+            std::vector<BuildableReq> subs;
+            for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p});
             buildPaths(subs);
         } catch (Error & e) {
             logWarning(e.info());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index da7ac4460..59d0983df 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -2,7 +2,7 @@
 
 #include "realisation.hh"
 #include "path.hh"
-#include "path-with-outputs.hh"
+#include "buildable.hh"
 #include "hash.hh"
 #include "content-address.hh"
 #include "serialise.hh"
@@ -494,7 +494,7 @@ public:
        recursively building any sub-derivations. For inputs that are
        not derivations, substitute them. */
     virtual void buildPaths(
-        const std::vector<StorePathWithOutputs> & paths,
+        const std::vector<BuildableReq> & paths,
         BuildMode buildMode = bmNormal);
 
     /* Build a single non-materialized derivation (i.e. not from an
@@ -656,7 +656,7 @@ public:
     /* Given a set of paths that are to be built, return the set of
        derivations that will be built, and the set of output paths
        that will be substituted. */
-    virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+    virtual void queryMissing(const std::vector<BuildableReq> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize);
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index be071dd78..0255726ac 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -86,9 +86,11 @@ namespace worker_proto {
 MAKE_WORKER_PROTO(, std::string);
 MAKE_WORKER_PROTO(, StorePath);
 MAKE_WORKER_PROTO(, ContentAddress);
+MAKE_WORKER_PROTO(, BuildableReq);
 MAKE_WORKER_PROTO(, Realisation);
 MAKE_WORKER_PROTO(, DrvOutput);
 
+MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
 MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
 
 #define X_ template<typename K, typename V>
@@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_);
 MAKE_WORKER_PROTO(, std::optional<StorePath>);
 MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
 
+template<typename T>
+std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
+{
+    std::vector<T> resSet;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        resSet.push_back(read(store, from, Phantom<T> {}));
+    }
+    return resSet;
+}
+
+template<typename T>
+void write(const Store & store, Sink & out, const std::vector<T> & resSet)
+{
+    out << resSet.size();
+    for (auto & key : resSet) {
+        write(store, out, key);
+    }
+}
+
 template<typename T>
 std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
 {
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 65b85b304..6f8a61261 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -12,6 +12,7 @@
 #include "affinity.hh"
 #include "util.hh"
 #include "shared.hh"
+#include "path-with-outputs.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "get-drvs.hh"
@@ -321,7 +322,8 @@ static void main_nix_build(int argc, char * * argv)
 
     state->printStats();
 
-    auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) {
+    auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths0) {
+        auto paths = toBuildableReqs(paths0);
         /* Note: we do this even when !printMissing to efficiently
            fetch binary cache data. */
         uint64_t downloadSize, narSize;
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 0f10a4cbb..af1c69b87 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -6,6 +6,7 @@
 #include "globals.hh"
 #include "names.hh"
 #include "profiles.hh"
+#include "path-with-outputs.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
@@ -418,13 +419,13 @@ static void queryInstSources(EvalState & state,
 
 static void printMissing(EvalState & state, DrvInfos & elems)
 {
-    std::vector<StorePathWithOutputs> targets;
+    std::vector<BuildableReq> targets;
     for (auto & i : elems) {
         Path drvPath = i.queryDrvPath();
         if (drvPath != "")
-            targets.push_back({state.store->parseStorePath(drvPath)});
+            targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)});
         else
-            targets.push_back({state.store->parseStorePath(i.queryOutPath())});
+            targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())});
     }
 
     printMissing(state.store, targets);
@@ -693,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     if (globals.forceName != "")
         drv.setName(globals.forceName);
 
-    if (drv.queryDrvPath() != "") {
-        std::vector<StorePathWithOutputs> paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}};
-        printMissing(globals.state->store, paths);
-        if (globals.dryRun) return;
-        globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
-    } else {
-        printMissing(globals.state->store,
-            {{globals.state->store->parseStorePath(drv.queryOutPath())}});
-        if (globals.dryRun) return;
-        globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath()));
-    }
+    std::vector<BuildableReq> paths {
+        (drv.queryDrvPath() != "")
+        ? (BuildableReq) (BuildableReqFromDrv {
+                globals.state->store->parseStorePath(drv.queryDrvPath())
+            })
+        : (BuildableReq) (BuildableOpaque {
+                globals.state->store->parseStorePath(drv.queryOutPath())
+            }),
+    };
+    printMissing(globals.state->store, paths);
+    if (globals.dryRun) return;
+    globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
 
     debug(format("switching to new user environment"));
     Path generation = createGeneration(
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 168ac492b..0ccf960fb 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "path-with-outputs.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
 #include "shared.hh"
@@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
             drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())});
 
     debug(format("building user environment dependencies"));
-    state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal);
+    state.store->buildPaths(
+        toBuildableReqs(drvsToBuild),
+        state.repair ? bmRepair : bmNormal);
 
     /* Construct the whole top level derivation. */
     StorePathSet references;
@@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     debug("building user environment");
     std::vector<StorePathWithOutputs> topLevelDrvs;
     topLevelDrvs.push_back({topLevelDrv});
-    state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal);
+    state.store->buildPaths(
+        toBuildableReqs(topLevelDrvs),
+        state.repair ? bmRepair : bmNormal);
 
     /* Switch the current user environment to the output path. */
     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index bfd1299fc..21c1e547b 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -10,6 +10,7 @@
 #include "worker-protocol.hh"
 #include "graphml.hh"
 #include "legacy.hh"
+#include "path-with-outputs.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -62,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
     auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
 
     if (path.path.isDerivation()) {
-        if (build) store->buildPaths({path});
+        if (build) store->buildPaths({path.toBuildableReq()});
         auto outputPaths = store->queryDerivationOutputMap(path.path);
         Derivation drv = store->derivationFromPath(path.path);
         rootNr++;
@@ -132,7 +133,9 @@ static void opRealise(Strings opFlags, Strings opArgs)
 
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
-    store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+    store->queryMissing(
+        toBuildableReqs(paths),
+        willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (ignoreUnknown) {
         std::vector<StorePathWithOutputs> paths2;
@@ -148,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
     if (dryRun) return;
 
     /* Build all paths at the same time to exploit parallelism. */
-    store->buildPaths(paths, buildMode);
+    store->buildPaths(toBuildableReqs(paths), buildMode);
 
     if (!ignoreUnknown)
         for (auto & i : paths) {
@@ -879,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 try {
                     MonitorFdHup monitor(in.fd);
-                    store->buildPaths(paths);
+                    store->buildPaths(toBuildableReqs(paths));
                     out << 0;
                 } catch (Error & e) {
                     assert(e.status);
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 48f4eb6e3..e86fbb3f7 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand
         auto evalState = getEvalState();
 
         auto app = installable->toApp(*evalState);
-        store->buildPaths(app.context);
+        store->buildPaths(toBuildableReqs(app.context));
 
         auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
@@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand
 
         StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
 
-        store->buildPaths({{drvPath}});
+        store->buildPaths({ BuildableReqFromDrv { drvPath } });
 
         auto outPathS = store->printStorePath(outPath);
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index d0b140570..616e2073e 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -3,6 +3,7 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "path-with-outputs.hh"
 #include "derivations.hh"
 #include "affinity.hh"
 #include "progress-bar.hh"
@@ -159,7 +160,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     auto shellDrvPath = writeDerivation(*store, drv);
 
     /* Build the derivation. */
-    store->buildPaths({{shellDrvPath}});
+    store->buildPaths({BuildableReqFromDrv{shellDrvPath}});
 
     for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
         auto & [_1, optPath] = outputAndOptPath;
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index a2b6c0303..9d6d22a43 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -7,6 +7,7 @@
 #include "get-drvs.hh"
 #include "store-api.hh"
 #include "derivations.hh"
+#include "path-with-outputs.hh"
 #include "attr-path.hh"
 #include "fetchers.hh"
 #include "registry.hh"
@@ -292,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand
             }
         };
 
-        std::vector<StorePathWithOutputs> drvPaths;
+        std::vector<BuildableReq> drvPaths;
 
         auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) {
             try {
@@ -461,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand
                                         fmt("%s.%s.%s", name, attr.name, attr2.name),
                                         *attr2.value, *attr2.pos);
                                     if ((std::string) attr.name == settings.thisSystem.get())
-                                        drvPaths.push_back({drvPath});
+                                        drvPaths.push_back(BuildableReqFromDrv{drvPath});
                                 }
                             }
                         }
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 4d275f577..b96e71844 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
     {
         ProfileManifest manifest(*getEvalState(), *profile);
 
-        std::vector<StorePathWithOutputs> pathsToBuild;
+        std::vector<BuildableReq> pathsToBuild;
 
         for (auto & installable : installables) {
             if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
@@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     attrPath,
                 };
 
-                pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}});
+                pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}});
 
                 manifest.elements.emplace_back(std::move(element));
             } else {
@@ -260,12 +260,15 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 
                     std::visit(overloaded {
                         [&](BuildableOpaque bo) {
-                            pathsToBuild.push_back({bo.path, {}});
+                            pathsToBuild.push_back(bo);
                             element.storePaths.insert(bo.path);
                         },
                         [&](BuildableFromDrv bfd) {
+                            // TODO: Why are we querying if we know the output
+                            // names already? Is it just to figure out what the
+                            // default one is?
                             for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
-                                pathsToBuild.push_back({bfd.drvPath, {output.first}});
+                                pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}});
                                 element.storePaths.insert(output.second);
                             }
                         },
@@ -388,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
         auto matchers = getMatchers(store);
 
         // FIXME: code duplication
-        std::vector<StorePathWithOutputs> pathsToBuild;
+        std::vector<BuildableReq> pathsToBuild;
 
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
@@ -423,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                     attrPath,
                 };
 
-                pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
+                pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME
             }
         }
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ec9388234..2e9bb41cc 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon
 
         auto app = installable->toApp(*state);
 
-        state->store->buildPaths(app.context);
+        state->store->buildPaths(toBuildableReqs(app.context));
 
         Strings allArgs{app.program};
         for (auto & i : args) allArgs.push_back(i);

From 4fe41c6db390c0295d20f6365ebedaec8ec79e1d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 5 Apr 2021 09:15:25 -0400
Subject: [PATCH 842/882] No templates for `Buildable` and `BuildableReq`

---
 src/libstore/buildable.cc |  3 ---
 src/libstore/buildable.hh | 30 +++++++++++++++++-------------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc
index 63ca1779e..7892b94e4 100644
--- a/src/libstore/buildable.cc
+++ b/src/libstore/buildable.cc
@@ -11,7 +11,6 @@ nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
     return res;
 }
 
-template<>
 nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
     nlohmann::json res;
     res["drvPath"] = store->printStorePath(drvPath);
@@ -36,7 +35,6 @@ std::string BuildableOpaque::to_string(const Store & store) const {
     return store.printStorePath(path);
 }
 
-template<>
 std::string BuildableReqFromDrv::to_string(const Store & store) const {
     return store.printStorePath(drvPath)
         + "!"
@@ -56,7 +54,6 @@ BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s)
     return {store.parseStorePath(s)};
 }
 
-template<>
 BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s)
 {
     size_t n = s.find("!");
diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh
index db78316bd..54e627271 100644
--- a/src/libstore/buildable.hh
+++ b/src/libstore/buildable.hh
@@ -20,31 +20,35 @@ struct BuildableOpaque {
     static BuildableOpaque parse(const Store & store, std::string_view);
 };
 
-template<typename Outputs>
-struct BuildableForFromDrv {
+struct BuildableReqFromDrv {
     StorePath drvPath;
-    Outputs outputs;
+    std::set<std::string> outputs;
 
-    nlohmann::json toJSON(ref<Store> store) const;
     std::string to_string(const Store & store) const;
-    static BuildableForFromDrv<Outputs> parse(const Store & store, std::string_view);
+    static BuildableReqFromDrv parse(const Store & store, std::string_view);
 };
 
-template <typename Outputs>
-using BuildableFor = std::variant<
+using BuildableReq = std::variant<
     BuildableOpaque,
-    BuildableForFromDrv<Outputs>
+    BuildableReqFromDrv
 >;
 
-typedef BuildableForFromDrv<std::set<std::string>> BuildableReqFromDrv;
-typedef BuildableFor<std::set<std::string>> BuildableReq;
-
 std::string to_string(const Store & store, const BuildableReq &);
 
 BuildableReq parseBuildableReq(const Store & store, std::string_view);
 
-typedef BuildableForFromDrv<std::map<std::string, std::optional<StorePath>>> BuildableFromDrv;
-typedef BuildableFor<std::map<std::string, std::optional<StorePath>>> Buildable;
+struct BuildableFromDrv {
+    StorePath drvPath;
+    std::map<std::string, std::optional<StorePath>> outputs;
+
+    nlohmann::json toJSON(ref<Store> store) const;
+    static BuildableFromDrv parse(const Store & store, std::string_view);
+};
+
+using Buildable = std::variant<
+    BuildableOpaque,
+    BuildableFromDrv
+>;
 
 typedef std::vector<Buildable> Buildables;
 

From 9dfb97c987d8b9d6a3d15f016e40f22f91deb764 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 5 Apr 2021 09:24:42 -0400
Subject: [PATCH 843/882] "newtype" BuildableReq

This makes for better types errors and allows us to give it methods.
---
 src/libstore/build/derivation-goal.cc       |  4 ++--
 src/libstore/build/entry-points.cc          |  2 +-
 src/libstore/build/local-derivation-goal.cc |  4 ++--
 src/libstore/buildable.cc                   |  6 +++---
 src/libstore/buildable.hh                   | 14 +++++++++++---
 src/libstore/misc.cc                        |  4 ++--
 src/libstore/path-with-outputs.cc           |  2 +-
 src/libstore/remote-store.cc                |  4 ++--
 8 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 8680d0bce..8396abbcd 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
-        to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs }));
+        BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
     state = &DerivationGoal::haveDerivation;
     name = fmt(
         "building of '%s' from in-memory derivation",
-        to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() }));
+        BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index d1973d78b..fc6294545 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -19,7 +19,7 @@ void Store::buildPaths(const std::vector<BuildableReq> & reqs, BuildMode buildMo
             [&](BuildableOpaque bo) {
                 goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
             },
-        }, br);
+        }, br.raw());
     }
 
     worker.run(goals);
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index c245527c9..6cc384719 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1200,7 +1200,7 @@ static StorePath pathPartOfReq(const BuildableReq & req)
         [&](BuildableReqFromDrv bfd)  {
             return bfd.drvPath;
         },
-    }, req);
+    }, req.raw());
 }
 
 
@@ -1340,7 +1340,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 
         for (auto & req : paths) {
             if (!goal.isAllowed(req))
-                throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req));
+                throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
         }
 
         next->buildPaths(paths, buildMode);
diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc
index 7892b94e4..31fef2faa 100644
--- a/src/libstore/buildable.cc
+++ b/src/libstore/buildable.cc
@@ -41,11 +41,11 @@ std::string BuildableReqFromDrv::to_string(const Store & store) const {
         + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
 }
 
-std::string to_string(const Store & store, const BuildableReq & req)
+std::string BuildableReq::to_string(const Store & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
-        req);
+        this->raw());
 }
 
 
@@ -66,7 +66,7 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_
     return {drvPath, outputs};
 }
 
-BuildableReq parseBuildableReq(const Store & store, std::string_view s)
+BuildableReq BuildableReq::parse(const Store & store, std::string_view s)
 {
     size_t n = s.find("!");
     return n == s.npos
diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh
index 54e627271..8317f3995 100644
--- a/src/libstore/buildable.hh
+++ b/src/libstore/buildable.hh
@@ -28,14 +28,22 @@ struct BuildableReqFromDrv {
     static BuildableReqFromDrv parse(const Store & store, std::string_view);
 };
 
-using BuildableReq = std::variant<
+using _BuildableReqRaw = std::variant<
     BuildableOpaque,
     BuildableReqFromDrv
 >;
 
-std::string to_string(const Store & store, const BuildableReq &);
+struct BuildableReq : _BuildableReqRaw {
+    using Raw = _BuildableReqRaw;
+    using Raw::Raw;
 
-BuildableReq parseBuildableReq(const Store & store, std::string_view);
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    std::string to_string(const Store & store) const;
+    static BuildableReq parse(const Store & store, std::string_view);
+};
 
 struct BuildableFromDrv {
     StorePath drvPath;
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index e702a4f9e..abfae1502 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -187,7 +187,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
 
         {
             auto state(state_.lock());
-            if (!state->done.insert(to_string(*this, req)).second) return;
+            if (!state->done.insert(req.to_string(*this)).second) return;
         }
 
         std::visit(overloaded {
@@ -250,7 +250,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
             for (auto & ref : info->second.references)
                 pool.enqueue(std::bind(doPath, BuildableOpaque { ref }));
           },
-        }, req);
+        }, req.raw());
     };
 
     for (auto & path : targets)
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
index 353286ac6..2898b8d4f 100644
--- a/src/libstore/path-with-outputs.cc
+++ b/src/libstore/path-with-outputs.cc
@@ -41,7 +41,7 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromBuild
         [&](BuildableReqFromDrv bfd) -> std::variant<StorePathWithOutputs, StorePath> {
             return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
         },
-    }, p);
+    }, p.raw());
 }
 
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index de1c95ed6..cb6402213 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -55,12 +55,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
 BuildableReq read(const Store & store, Source & from, Phantom<BuildableReq> _)
 {
     auto s = readString(from);
-    return parseBuildableReq(store, s);
+    return BuildableReq::parse(store, s);
 }
 
 void write(const Store & store, Sink & out, const BuildableReq & req)
 {
-    out << to_string(store, req);
+    out << req.to_string(store);
 }
 
 

From 9b805d36ac70545fc4c0d863e21e0c2e5f2518a1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 5 Apr 2021 09:48:18 -0400
Subject: [PATCH 844/882] Rename Buildable

---
 src/libcmd/command.cc                       |  6 +--
 src/libcmd/command.hh                       |  4 +-
 src/libcmd/installables.cc                  | 44 ++++++++++-----------
 src/libcmd/installables.hh                  |  6 +--
 src/libexpr/primops.cc                      |  6 +--
 src/libmain/shared.cc                       |  2 +-
 src/libmain/shared.hh                       |  2 +-
 src/libstore/build/derivation-goal.cc       |  4 +-
 src/libstore/build/entry-points.cc          |  6 +--
 src/libstore/build/local-derivation-goal.cc | 16 ++++----
 src/libstore/build/local-derivation-goal.hh |  2 +-
 src/libstore/build/worker.cc                |  6 +--
 src/libstore/buildable.cc                   | 24 +++++------
 src/libstore/buildable.hh                   | 37 +++++++++--------
 src/libstore/daemon.cc                      | 12 +++---
 src/libstore/legacy-ssh-store.cc            |  4 +-
 src/libstore/misc.cc                        | 16 ++++----
 src/libstore/path-with-outputs.cc           | 18 ++++-----
 src/libstore/path-with-outputs.hh           |  6 +--
 src/libstore/remote-store.cc                | 18 ++++-----
 src/libstore/remote-store.hh                |  4 +-
 src/libstore/store-api.cc                   |  8 ++--
 src/libstore/store-api.hh                   |  4 +-
 src/libstore/worker-protocol.hh             |  2 +-
 src/nix-build/nix-build.cc                  |  2 +-
 src/nix-env/nix-env.cc                      | 12 +++---
 src/nix-env/user-env.cc                     |  4 +-
 src/nix-store/nix-store.cc                  |  8 ++--
 src/nix/build.cc                            |  6 +--
 src/nix/bundle.cc                           |  4 +-
 src/nix/develop.cc                          |  8 ++--
 src/nix/flake.cc                            |  4 +-
 src/nix/log.cc                              |  6 +--
 src/nix/profile.cc                          | 14 +++----
 src/nix/run.cc                              |  2 +-
 35 files changed, 165 insertions(+), 162 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index d29954f67..dc1fbc43f 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -162,7 +162,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
             profile2, storePath));
 }
 
-void MixProfile::updateProfile(const Buildables & buildables)
+void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
 {
     if (!profile) return;
 
@@ -170,10 +170,10 @@ void MixProfile::updateProfile(const Buildables & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](BuildableOpaque bo) {
+            [&](DerivedPathOpaque bo) {
                 result.push_back(bo.path);
             },
-            [&](BuildableFromDrv bfd) {
+            [&](DerivedPathWithHintsBuilt bfd) {
                 for (auto & output : bfd.outputs) {
                     /* Output path should be known because we just tried to
                        build it. */
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index e66c697eb..9e18c6e51 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-Buildables build(ref<Store> store, Realise mode,
+DerivedPathsWithHints build(ref<Store> store, Realise mode,
     std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
 
 std::set<StorePath> toStorePaths(ref<Store> store,
@@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand
 
     /* If 'profile' is set, make it point at the store path produced
        by 'buildables'. */
-    void updateProfile(const Buildables & buildables);
+    void updateProfile(const DerivedPathsWithHints & buildables);
 };
 
 struct MixDefaultProfile : MixProfile
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index b68c5f6a7..f091ac186 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -285,9 +285,9 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
     }
 }
 
-Buildable Installable::toBuildable()
+DerivedPathWithHints Installable::toDerivedPathWithHints()
 {
-    auto buildables = toBuildables();
+    auto buildables = toDerivedPathsWithHints();
     if (buildables.size() != 1)
         throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
     return std::move(buildables[0]);
@@ -321,7 +321,7 @@ struct InstallableStorePath : Installable
 
     std::string what() override { return store->printStorePath(storePath); }
 
-    Buildables toBuildables() override
+    DerivedPathsWithHints toDerivedPathsWithHints() override
     {
         if (storePath.isDerivation()) {
             std::map<std::string, std::optional<StorePath>> outputs;
@@ -329,14 +329,14 @@ struct InstallableStorePath : Installable
             for (auto & [name, output] : drv.outputsAndOptPaths(*store))
                 outputs.emplace(name, output.second);
             return {
-                BuildableFromDrv {
+                DerivedPathWithHintsBuilt {
                     .drvPath = storePath,
                     .outputs = std::move(outputs)
                 }
             };
         } else {
             return {
-                BuildableOpaque {
+                DerivedPathOpaque {
                     .path = storePath,
                 }
             };
@@ -349,9 +349,9 @@ struct InstallableStorePath : Installable
     }
 };
 
-Buildables InstallableValue::toBuildables()
+DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints()
 {
-    Buildables res;
+    DerivedPathsWithHints res;
 
     std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
 
@@ -364,7 +364,7 @@ Buildables InstallableValue::toBuildables()
     }
 
     for (auto & i : drvsToOutputs)
-        res.push_back(BuildableFromDrv { i.first, i.second });
+        res.push_back(DerivedPathWithHintsBuilt { i.first, i.second });
 
     return res;
 }
@@ -671,28 +671,28 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-Buildables build(ref<Store> store, Realise mode,
+DerivedPathsWithHints build(ref<Store> store, Realise mode,
     std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
 
-    Buildables buildables;
+    DerivedPathsWithHints buildables;
 
-    std::vector<BuildableReq> pathsToBuild;
+    std::vector<DerivedPath> pathsToBuild;
 
     for (auto & i : installables) {
-        for (auto & b : i->toBuildables()) {
+        for (auto & b : i->toDerivedPathsWithHints()) {
             std::visit(overloaded {
-                [&](BuildableOpaque bo) {
+                [&](DerivedPathOpaque bo) {
                     pathsToBuild.push_back(bo);
                 },
-                [&](BuildableFromDrv bfd) {
+                [&](DerivedPathWithHintsBuilt bfd) {
                     StringSet outputNames;
                     for (auto & output : bfd.outputs)
                         outputNames.insert(output.first);
                     pathsToBuild.push_back(
-                        BuildableReqFromDrv{bfd.drvPath, outputNames});
+                        DerivedPath::Built{bfd.drvPath, outputNames});
                 },
             }, b);
             buildables.push_back(std::move(b));
@@ -717,10 +717,10 @@ std::set<RealisedPath> toRealisedPaths(
     if (operateOn == OperateOn::Output) {
         for (auto & b : build(store, mode, installables))
             std::visit(overloaded {
-                [&](BuildableOpaque bo) {
+                [&](DerivedPathOpaque bo) {
                     res.insert(bo.path);
                 },
-                [&](BuildableFromDrv bfd) {
+                [&](DerivedPathWithHintsBuilt bfd) {
                     auto drv = store->readDerivation(bfd.drvPath);
                     auto outputHashes = staticOutputHashes(*store, drv);
                     for (auto & output : bfd.outputs) {
@@ -751,8 +751,8 @@ std::set<RealisedPath> toRealisedPaths(
             settings.readOnlyMode = true;
 
         for (auto & i : installables)
-            for (auto & b : i->toBuildables())
-                if (auto bfd = std::get_if<BuildableFromDrv>(&b))
+            for (auto & b : i->toDerivedPathsWithHints())
+                if (auto bfd = std::get_if<DerivedPathWithHintsBuilt>(&b))
                     res.insert(bfd->drvPath);
     }
 
@@ -787,9 +787,9 @@ StorePathSet toDerivations(ref<Store> store,
     StorePathSet drvPaths;
 
     for (auto & i : installables)
-        for (auto & b : i->toBuildables())
+        for (auto & b : i->toDerivedPathsWithHints())
             std::visit(overloaded {
-                [&](BuildableOpaque bo) {
+                [&](DerivedPathOpaque bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
                     auto derivers = store->queryValidDerivers(bo.path);
@@ -798,7 +798,7 @@ StorePathSet toDerivations(ref<Store> store,
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
                 },
-                [&](BuildableFromDrv bfd) {
+                [&](DerivedPathWithHintsBuilt bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
             }, b);
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index e5c6fe208..0bc932b52 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -29,9 +29,9 @@ struct Installable
 
     virtual std::string what() = 0;
 
-    virtual Buildables toBuildables() = 0;
+    virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0;
 
-    Buildable toBuildable();
+    DerivedPathWithHints toDerivedPathWithHints();
 
     App toApp(EvalState & state);
 
@@ -74,7 +74,7 @@ struct InstallableValue : Installable
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
-    Buildables toBuildables() override;
+    DerivedPathsWithHints toDerivedPathsWithHints() override;
 };
 
 struct InstallableFlake : InstallableValue
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 24bc34b74..428adf4c2 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) :
 
 void EvalState::realiseContext(const PathSet & context)
 {
-    std::vector<BuildableReqFromDrv> drvs;
+    std::vector<DerivedPath::Built> drvs;
 
     for (auto & i : context) {
         auto [ctxS, outputName] = decodeContext(i);
@@ -56,8 +56,8 @@ void EvalState::realiseContext(const PathSet & context)
     /* For performance, prefetch all substitute info. */
     StorePathSet willBuild, willSubstitute, unknown;
     uint64_t downloadSize, narSize;
-    std::vector<BuildableReq> buildReqs;
-    for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d });
+    std::vector<DerivedPath> buildReqs;
+    for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
     store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     store->buildPaths(buildReqs);
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 20027e099..09af57871 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -36,7 +36,7 @@ void printGCWarning()
 }
 
 
-void printMissing(ref<Store> store, const std::vector<BuildableReq> & paths, Verbosity lvl)
+void printMissing(ref<Store> store, const std::vector<DerivedPath> & paths, Verbosity lvl)
 {
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 18e0fb57d..9cb9e6da2 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -43,7 +43,7 @@ struct StorePathWithOutputs;
 
 void printMissing(
     ref<Store> store,
-    const std::vector<BuildableReq> & paths,
+    const std::vector<DerivedPath> & paths,
     Verbosity lvl = lvlInfo);
 
 void printMissing(ref<Store> store, const StorePathSet & willBuild,
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 8396abbcd..3ce538f77 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
-        BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store));
+        DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
     state = &DerivationGoal::haveDerivation;
     name = fmt(
         "building of '%s' from in-memory derivation",
-        BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store));
+        DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index fc6294545..732d4785d 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -6,17 +6,17 @@
 
 namespace nix {
 
-void Store::buildPaths(const std::vector<BuildableReq> & reqs, BuildMode buildMode)
+void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
 {
     Worker worker(*this);
 
     Goals goals;
     for (auto & br : reqs) {
         std::visit(overloaded {
-            [&](BuildableReqFromDrv bfd) {
+            [&](DerivedPath::Built bfd) {
                 goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
             },
-            [&](BuildableOpaque bo) {
+            [&](DerivedPath::Opaque bo) {
                 goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
             },
         }, br.raw());
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 6cc384719..7c1402918 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1191,20 +1191,20 @@ void LocalDerivationGoal::writeStructuredAttrs()
 }
 
 
-static StorePath pathPartOfReq(const BuildableReq & req)
+static StorePath pathPartOfReq(const DerivedPath & req)
 {
     return std::visit(overloaded {
-        [&](BuildableOpaque bo) {
+        [&](DerivedPath::Opaque bo) {
             return bo.path;
         },
-        [&](BuildableReqFromDrv bfd)  {
+        [&](DerivedPath::Built bfd)  {
             return bfd.drvPath;
         },
     }, req.raw());
 }
 
 
-bool LocalDerivationGoal::isAllowed(const BuildableReq & req)
+bool LocalDerivationGoal::isAllowed(const DerivedPath & req)
 {
     return this->isAllowed(pathPartOfReq(req));
 }
@@ -1332,7 +1332,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     // an allowed derivation
     { throw Error("queryRealisation"); }
 
-    void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override
+    void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
@@ -1346,7 +1346,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
         next->buildPaths(paths, buildMode);
 
         for (auto & path : paths) {
-            auto p =  std::get_if<BuildableReqFromDrv>(&path);
+            auto p =  std::get_if<DerivedPath::Built>(&path);
             if (!p) continue;
             auto & bfd = *p;
             auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
@@ -1380,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override
     { unsupported("addSignatures"); }
 
-    void queryMissing(const std::vector<BuildableReq> & targets,
+    void queryMissing(const std::vector<DerivedPath> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override
     {
@@ -1388,7 +1388,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
            client about what paths will be built/substituted or are
            already present. Probably not a big deal. */
 
-        std::vector<BuildableReq> allowed;
+        std::vector<DerivedPath> allowed;
         for (auto & req : targets) {
             if (goal.isAllowed(req))
                 allowed.emplace_back(req);
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index edb93f84e..d30be2351 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -116,7 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
-    bool isAllowed(const BuildableReq & req);
+    bool isAllowed(const DerivedPath & req);
 
     friend struct RestrictedStore;
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index fef4cb0cb..6c04d3ed3 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal)
 
 void Worker::run(const Goals & _topGoals)
 {
-    std::vector<nix::BuildableReq> topPaths;
+    std::vector<nix::DerivedPath> topPaths;
 
     for (auto & i : _topGoals) {
         topGoals.insert(i);
         if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
-            topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs});
+            topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs});
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
-            topPaths.push_back(BuildableOpaque{goal->storePath});
+            topPaths.push_back(DerivedPath::Opaque{goal->storePath});
         }
     }
 
diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc
index 31fef2faa..a8c0c70b1 100644
--- a/src/libstore/buildable.cc
+++ b/src/libstore/buildable.cc
@@ -5,13 +5,13 @@
 
 namespace nix {
 
-nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
+nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
     nlohmann::json res;
     res["path"] = store->printStorePath(path);
     return res;
 }
 
-nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
+nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref<Store> store) const {
     nlohmann::json res;
     res["drvPath"] = store->printStorePath(drvPath);
     for (const auto& [output, path] : outputs) {
@@ -20,9 +20,9 @@ nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
     return res;
 }
 
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
+nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store) {
     auto res = nlohmann::json::array();
-    for (const Buildable & buildable : buildables) {
+    for (const DerivedPathWithHints & buildable : buildables) {
         std::visit([&res, store](const auto & buildable) {
             res.push_back(buildable.toJSON(store));
         }, buildable);
@@ -31,17 +31,17 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store)
 }
 
 
-std::string BuildableOpaque::to_string(const Store & store) const {
+std::string DerivedPath::Opaque::to_string(const Store & store) const {
     return store.printStorePath(path);
 }
 
-std::string BuildableReqFromDrv::to_string(const Store & store) const {
+std::string DerivedPath::Built::to_string(const Store & store) const {
     return store.printStorePath(drvPath)
         + "!"
         + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
 }
 
-std::string BuildableReq::to_string(const Store & store) const
+std::string DerivedPath::to_string(const Store & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
@@ -49,12 +49,12 @@ std::string BuildableReq::to_string(const Store & store) const
 }
 
 
-BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s)
+DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
 {
     return {store.parseStorePath(s)};
 }
 
-BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s)
+DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s)
 {
     size_t n = s.find("!");
     assert(n != s.npos);
@@ -66,12 +66,12 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_
     return {drvPath, outputs};
 }
 
-BuildableReq BuildableReq::parse(const Store & store, std::string_view s)
+DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
 {
     size_t n = s.find("!");
     return n == s.npos
-        ? (BuildableReq) BuildableOpaque::parse(store, s)
-        : (BuildableReq) BuildableReqFromDrv::parse(store, s);
+        ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
+        : (DerivedPath) DerivedPath::Built::parse(store, s);
 }
 
 }
diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh
index 8317f3995..0a0cf8105 100644
--- a/src/libstore/buildable.hh
+++ b/src/libstore/buildable.hh
@@ -12,54 +12,57 @@ namespace nix {
 
 class Store;
 
-struct BuildableOpaque {
+struct DerivedPathOpaque {
     StorePath path;
 
     nlohmann::json toJSON(ref<Store> store) const;
     std::string to_string(const Store & store) const;
-    static BuildableOpaque parse(const Store & store, std::string_view);
+    static DerivedPathOpaque parse(const Store & store, std::string_view);
 };
 
-struct BuildableReqFromDrv {
+struct DerivedPathBuilt {
     StorePath drvPath;
     std::set<std::string> outputs;
 
     std::string to_string(const Store & store) const;
-    static BuildableReqFromDrv parse(const Store & store, std::string_view);
+    static DerivedPathBuilt parse(const Store & store, std::string_view);
 };
 
-using _BuildableReqRaw = std::variant<
-    BuildableOpaque,
-    BuildableReqFromDrv
+using _DerivedPathRaw = std::variant<
+    DerivedPathOpaque,
+    DerivedPathBuilt
 >;
 
-struct BuildableReq : _BuildableReqRaw {
-    using Raw = _BuildableReqRaw;
+struct DerivedPath : _DerivedPathRaw {
+    using Raw = _DerivedPathRaw;
     using Raw::Raw;
 
+    using Opaque = DerivedPathOpaque;
+    using Built = DerivedPathBuilt;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }
 
     std::string to_string(const Store & store) const;
-    static BuildableReq parse(const Store & store, std::string_view);
+    static DerivedPath parse(const Store & store, std::string_view);
 };
 
-struct BuildableFromDrv {
+struct DerivedPathWithHintsBuilt {
     StorePath drvPath;
     std::map<std::string, std::optional<StorePath>> outputs;
 
     nlohmann::json toJSON(ref<Store> store) const;
-    static BuildableFromDrv parse(const Store & store, std::string_view);
+    static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view);
 };
 
-using Buildable = std::variant<
-    BuildableOpaque,
-    BuildableFromDrv
+using DerivedPathWithHints = std::variant<
+    DerivedPath::Opaque,
+    DerivedPathWithHintsBuilt
 >;
 
-typedef std::vector<Buildable> Buildables;
+typedef std::vector<DerivedPathWithHints> DerivedPathsWithHints;
 
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
+nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store);
 
 }
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 6b527dcb2..affd60472 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -260,14 +260,14 @@ static void writeValidPathInfo(
     }
 }
 
-static std::vector<BuildableReq> readBuildableReqs(Store & store, unsigned int clientVersion, Source & from)
+static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from)
 {
-    std::vector<BuildableReq> reqs;
+    std::vector<DerivedPath> reqs;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
-        reqs = worker_proto::read(store, from, Phantom<std::vector<BuildableReq>> {});
+        reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
     } else {
         for (auto & s : readStrings<Strings>(from))
-            reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq());
+            reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
     }
     return reqs;
 }
@@ -506,7 +506,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopBuildPaths: {
-        auto drvs = readBuildableReqs(*store, clientVersion, from);
+        auto drvs = readDerivedPaths(*store, clientVersion, from);
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -870,7 +870,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryMissing: {
-        auto targets = readBuildableReqs(*store, clientVersion, from);
+        auto targets = readDerivedPaths(*store, clientVersion, from);
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         uint64_t downloadSize, narSize;
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 1cb977be6..edaf75136 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -267,14 +267,14 @@ public:
         return status;
     }
 
-    void buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode) override
+    void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode) override
     {
         auto conn(connections->get());
 
         conn->to << cmdBuildPaths;
         Strings ss;
         for (auto & p : drvPaths) {
-            auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
+            auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
             std::visit(overloaded {
                 [&](StorePathWithOutputs s) {
                     ss.push_back(s.to_string(*this));
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index abfae1502..a99a2fc78 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -117,7 +117,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
     return std::nullopt;
 }
 
-void Store::queryMissing(const std::vector<BuildableReq> & targets,
+void Store::queryMissing(const std::vector<DerivedPath> & targets,
     StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
     uint64_t & downloadSize_, uint64_t & narSize_)
 {
@@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
 
     Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
 
-    std::function<void(BuildableReq)> doPath;
+    std::function<void(DerivedPath)> doPath;
 
     auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
         {
@@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
         }
 
         for (auto & i : drv.inputDrvs)
-            pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second }));
+            pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second }));
     };
 
     auto checkOutput = [&](
@@ -177,13 +177,13 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
                 drvState->outPaths.insert(outPath);
                 if (!drvState->left) {
                     for (auto & path : drvState->outPaths)
-                        pool.enqueue(std::bind(doPath, BuildableOpaque { path } ));
+                        pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } ));
                 }
             }
         }
     };
 
-    doPath = [&](const BuildableReq & req) {
+    doPath = [&](const DerivedPath & req) {
 
         {
             auto state(state_.lock());
@@ -191,7 +191,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
         }
 
         std::visit(overloaded {
-          [&](BuildableReqFromDrv bfd) {
+          [&](DerivedPath::Built bfd) {
             if (!isValidPath(bfd.drvPath)) {
                 // FIXME: we could try to substitute the derivation.
                 auto state(state_.lock());
@@ -224,7 +224,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
                 mustBuildDrv(bfd.drvPath, *drv);
 
           },
-          [&](BuildableOpaque bo) {
+          [&](DerivedPath::Opaque bo) {
 
             if (isValidPath(bo.path)) return;
 
@@ -248,7 +248,7 @@ void Store::queryMissing(const std::vector<BuildableReq> & targets,
             }
 
             for (auto & ref : info->second.references)
-                pool.enqueue(std::bind(doPath, BuildableOpaque { ref }));
+                pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref }));
           },
         }, req.raw());
     };
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
index 2898b8d4f..865d64cf2 100644
--- a/src/libstore/path-with-outputs.cc
+++ b/src/libstore/path-with-outputs.cc
@@ -11,34 +11,34 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
 }
 
 
-BuildableReq StorePathWithOutputs::toBuildableReq() const
+DerivedPath StorePathWithOutputs::toDerivedPath() const
 {
     if (!outputs.empty() || path.isDerivation())
-        return BuildableReqFromDrv { path, outputs };
+        return DerivedPath::Built { path, outputs };
     else
-        return BuildableOpaque { path };
+        return DerivedPath::Opaque { path };
 }
 
 
-std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs> ss)
+std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss)
 {
-	std::vector<BuildableReq> reqs;
-	for (auto & s : ss) reqs.push_back(s.toBuildableReq());
+	std::vector<DerivedPath> reqs;
+	for (auto & s : ss) reqs.push_back(s.toDerivedPath());
 	return reqs;
 }
 
 
-std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p)
+std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
 {
     return std::visit(overloaded {
-        [&](BuildableOpaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
+        [&](DerivedPath::Opaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
             if (bo.path.isDerivation()) {
                 // drv path gets interpreted as "build", not "get drv file itself"
                 return bo.path;
             }
             return StorePathWithOutputs { bo.path };
         },
-        [&](BuildableReqFromDrv bfd) -> std::variant<StorePathWithOutputs, StorePath> {
+        [&](DerivedPath::Built bfd) -> std::variant<StorePathWithOutputs, StorePath> {
             return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
         },
     }, p.raw());
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
index 870cac08e..749348398 100644
--- a/src/libstore/path-with-outputs.hh
+++ b/src/libstore/path-with-outputs.hh
@@ -14,12 +14,12 @@ struct StorePathWithOutputs
 
     std::string to_string(const Store & store) const;
 
-    BuildableReq toBuildableReq() const;
+    DerivedPath toDerivedPath() const;
 
-    static std::variant<StorePathWithOutputs, StorePath> tryFromBuildableReq(const BuildableReq &);
+    static std::variant<StorePathWithOutputs, StorePath> tryFromDerivedPath(const DerivedPath &);
 };
 
-std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs>);
+std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>);
 
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index cb6402213..761b4a087 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -52,13 +52,13 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
 }
 
 
-BuildableReq read(const Store & store, Source & from, Phantom<BuildableReq> _)
+DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
 {
     auto s = readString(from);
-    return BuildableReq::parse(store, s);
+    return DerivedPath::parse(store, s);
 }
 
-void write(const Store & store, Sink & out, const BuildableReq & req)
+void write(const Store & store, Sink & out, const DerivedPath & req)
 {
     out << req.to_string(store);
 }
@@ -670,14 +670,14 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
     return {Realisation{.id = id, .outPath = *outPaths.begin()}};
 }
 
-static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector<BuildableReq> & reqs)
+static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
 {
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
         worker_proto::write(store, conn->to, reqs);
     } else {
         Strings ss;
         for (auto & p : reqs) {
-            auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
+            auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
             std::visit(overloaded {
                 [&](StorePathWithOutputs s) {
                     ss.push_back(s.to_string(store));
@@ -694,12 +694,12 @@ static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, con
     }
 }
 
-void RemoteStore::buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode)
+void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode)
 {
     auto conn(getConnection());
     conn->to << wopBuildPaths;
     assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
-    writeBuildableReqs(*this, conn, drvPaths);
+    writeDerivedPaths(*this, conn, drvPaths);
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
         conn->to << buildMode;
     else
@@ -838,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
 }
 
 
-void RemoteStore::queryMissing(const std::vector<BuildableReq> & targets,
+void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
     StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
     uint64_t & downloadSize, uint64_t & narSize)
 {
@@ -849,7 +849,7 @@ void RemoteStore::queryMissing(const std::vector<BuildableReq> & targets,
             // to prevent a deadlock.
             goto fallback;
         conn->to << wopQueryMissing;
-        writeBuildableReqs(*this, conn, targets);
+        writeDerivedPaths(*this, conn, targets);
         conn.processStderr();
         willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
         willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 20d366038..6cf76a46d 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -85,7 +85,7 @@ public:
 
     std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
 
-    void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override;
+    void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override;
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override;
@@ -108,7 +108,7 @@ public:
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
-    void queryMissing(const std::vector<BuildableReq> & targets,
+    void queryMissing(const std::vector<DerivedPath> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8b60bdc62..93fcb068f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath,
 
 void Store::substitutePaths(const StorePathSet & paths)
 {
-    std::vector<BuildableReq> paths2;
+    std::vector<DerivedPath> paths2;
     for (auto & path : paths)
         if (!path.isDerivation())
-            paths2.push_back(BuildableOpaque{path});
+            paths2.push_back(DerivedPath::Opaque{path});
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
     queryMissing(paths2,
@@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths)
 
     if (!willSubstitute.empty())
         try {
-            std::vector<BuildableReq> subs;
-            for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p});
+            std::vector<DerivedPath> subs;
+            for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p});
             buildPaths(subs);
         } catch (Error & e) {
             logWarning(e.info());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 59d0983df..483f3c5fa 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -494,7 +494,7 @@ public:
        recursively building any sub-derivations. For inputs that are
        not derivations, substitute them. */
     virtual void buildPaths(
-        const std::vector<BuildableReq> & paths,
+        const std::vector<DerivedPath> & paths,
         BuildMode buildMode = bmNormal);
 
     /* Build a single non-materialized derivation (i.e. not from an
@@ -656,7 +656,7 @@ public:
     /* Given a set of paths that are to be built, return the set of
        derivations that will be built, and the set of output paths
        that will be substituted. */
-    virtual void queryMissing(const std::vector<BuildableReq> & targets,
+    virtual void queryMissing(const std::vector<DerivedPath> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize);
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 0255726ac..001ed25e3 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -86,7 +86,7 @@ namespace worker_proto {
 MAKE_WORKER_PROTO(, std::string);
 MAKE_WORKER_PROTO(, StorePath);
 MAKE_WORKER_PROTO(, ContentAddress);
-MAKE_WORKER_PROTO(, BuildableReq);
+MAKE_WORKER_PROTO(, DerivedPath);
 MAKE_WORKER_PROTO(, Realisation);
 MAKE_WORKER_PROTO(, DrvOutput);
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 6f8a61261..d46bc1f2b 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -323,7 +323,7 @@ static void main_nix_build(int argc, char * * argv)
     state->printStats();
 
     auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths0) {
-        auto paths = toBuildableReqs(paths0);
+        auto paths = toDerivedPaths(paths0);
         /* Note: we do this even when !printMissing to efficiently
            fetch binary cache data. */
         uint64_t downloadSize, narSize;
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index af1c69b87..e04954d45 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -419,13 +419,13 @@ static void queryInstSources(EvalState & state,
 
 static void printMissing(EvalState & state, DrvInfos & elems)
 {
-    std::vector<BuildableReq> targets;
+    std::vector<DerivedPath> targets;
     for (auto & i : elems) {
         Path drvPath = i.queryDrvPath();
         if (drvPath != "")
-            targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)});
+            targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)});
         else
-            targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())});
+            targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())});
     }
 
     printMissing(state.store, targets);
@@ -694,12 +694,12 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     if (globals.forceName != "")
         drv.setName(globals.forceName);
 
-    std::vector<BuildableReq> paths {
+    std::vector<DerivedPath> paths {
         (drv.queryDrvPath() != "")
-        ? (BuildableReq) (BuildableReqFromDrv {
+        ? (DerivedPath) (DerivedPath::Built {
                 globals.state->store->parseStorePath(drv.queryDrvPath())
             })
-        : (BuildableReq) (BuildableOpaque {
+        : (DerivedPath) (DerivedPath::Opaque {
                 globals.state->store->parseStorePath(drv.queryOutPath())
             }),
     };
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 0ccf960fb..5ceb2ae67 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -43,7 +43,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
     debug(format("building user environment dependencies"));
     state.store->buildPaths(
-        toBuildableReqs(drvsToBuild),
+        toDerivedPaths(drvsToBuild),
         state.repair ? bmRepair : bmNormal);
 
     /* Construct the whole top level derivation. */
@@ -140,7 +140,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     std::vector<StorePathWithOutputs> topLevelDrvs;
     topLevelDrvs.push_back({topLevelDrv});
     state.store->buildPaths(
-        toBuildableReqs(topLevelDrvs),
+        toDerivedPaths(topLevelDrvs),
         state.repair ? bmRepair : bmNormal);
 
     /* Switch the current user environment to the output path. */
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 21c1e547b..b327793e7 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -63,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
     auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
 
     if (path.path.isDerivation()) {
-        if (build) store->buildPaths({path.toBuildableReq()});
+        if (build) store->buildPaths({path.toDerivedPath()});
         auto outputPaths = store->queryDerivationOutputMap(path.path);
         Derivation drv = store->derivationFromPath(path.path);
         rootNr++;
@@ -134,7 +134,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
     store->queryMissing(
-        toBuildableReqs(paths),
+        toDerivedPaths(paths),
         willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (ignoreUnknown) {
@@ -151,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
     if (dryRun) return;
 
     /* Build all paths at the same time to exploit parallelism. */
-    store->buildPaths(toBuildableReqs(paths), buildMode);
+    store->buildPaths(toDerivedPaths(paths), buildMode);
 
     if (!ignoreUnknown)
         for (auto & i : paths) {
@@ -882,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 try {
                     MonitorFdHup monitor(in.fd);
-                    store->buildPaths(toBuildableReqs(paths));
+                    store->buildPaths(toDerivedPaths(paths));
                     out << 0;
                 } catch (Error & e) {
                     assert(e.status);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 724ce9d79..0529ed382 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                 for (const auto & [_i, buildable] : enumerate(buildables)) {
                     auto i = _i;
                     std::visit(overloaded {
-                        [&](BuildableOpaque bo) {
+                        [&](DerivedPathOpaque bo) {
                             std::string symlink = outLink;
                             if (i) symlink += fmt("-%d", i);
                             store2->addPermRoot(bo.path, absPath(symlink));
                         },
-                        [&](BuildableFromDrv bfd) {
+                        [&](DerivedPathWithHintsBuilt bfd) {
                             auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
                             for (auto & output : builtOutputs) {
                                 std::string symlink = outLink;
@@ -80,7 +80,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         updateProfile(buildables);
 
-        if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump());
     }
 };
 
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index e86fbb3f7..53dccc63a 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand
         auto evalState = getEvalState();
 
         auto app = installable->toApp(*evalState);
-        store->buildPaths(toBuildableReqs(app.context));
+        store->buildPaths(toDerivedPaths(app.context));
 
         auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
@@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand
 
         StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
 
-        store->buildPaths({ BuildableReqFromDrv { drvPath } });
+        store->buildPaths({ DerivedPath::Built { drvPath } });
 
         auto outPathS = store->printStorePath(outPath);
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 616e2073e..cae6ded40 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -160,7 +160,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     auto shellDrvPath = writeDerivation(*store, drv);
 
     /* Build the derivation. */
-    store->buildPaths({BuildableReqFromDrv{shellDrvPath}});
+    store->buildPaths({DerivedPath::Built{shellDrvPath}});
 
     for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
         auto & [_1, optPath] = outputAndOptPath;
@@ -265,7 +265,7 @@ struct Common : InstallableCommand, MixProfile
         for (auto & [installable_, dir_] : redirects) {
             auto dir = absPath(dir_);
             auto installable = parseInstallable(store, installable_);
-            auto buildable = installable->toBuildable();
+            auto buildable = installable->toDerivedPathWithHints();
             auto doRedirect = [&](const StorePath & path)
             {
                 auto from = store->printStorePath(path);
@@ -277,10 +277,10 @@ struct Common : InstallableCommand, MixProfile
                 }
             };
             std::visit(overloaded {
-                [&](const BuildableOpaque & bo) {
+                [&](const DerivedPathOpaque & bo) {
                     doRedirect(bo.path);
                 },
-                [&](const BuildableFromDrv & bfd) {
+                [&](const DerivedPathWithHintsBuilt & bfd) {
                     for (auto & [outputName, path] : bfd.outputs)
                         if (path) doRedirect(*path);
                 },
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 9d6d22a43..62a413e27 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -293,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand
             }
         };
 
-        std::vector<BuildableReq> drvPaths;
+        std::vector<DerivedPath> drvPaths;
 
         auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) {
             try {
@@ -462,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand
                                         fmt("%s.%s.%s", name, attr.name, attr2.name),
                                         *attr2.value, *attr2.pos);
                                     if ((std::string) attr.name == settings.thisSystem.get())
-                                        drvPaths.push_back(BuildableReqFromDrv{drvPath});
+                                        drvPaths.push_back(DerivedPath::Built{drvPath});
                                 }
                             }
                         }
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 67d3742d6..5010e3326 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand
 
         subs.push_front(store);
 
-        auto b = installable->toBuildable();
+        auto b = installable->toDerivedPathWithHints();
 
         RunPager pager;
         for (auto & sub : subs) {
             auto log = std::visit(overloaded {
-                [&](BuildableOpaque bo) {
+                [&](DerivedPathOpaque bo) {
                     return sub->getBuildLog(bo.path);
                 },
-                [&](BuildableFromDrv bfd) {
+                [&](DerivedPathWithHintsBuilt bfd) {
                     return sub->getBuildLog(bfd.drvPath);
                 },
             }, b);
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index b96e71844..ad824dd70 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
     {
         ProfileManifest manifest(*getEvalState(), *profile);
 
-        std::vector<BuildableReq> pathsToBuild;
+        std::vector<DerivedPath> pathsToBuild;
 
         for (auto & installable : installables) {
             if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
@@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     attrPath,
                 };
 
-                pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}});
+                pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}});
 
                 manifest.elements.emplace_back(std::move(element));
             } else {
@@ -259,16 +259,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     ProfileElement element;
 
                     std::visit(overloaded {
-                        [&](BuildableOpaque bo) {
+                        [&](DerivedPathOpaque bo) {
                             pathsToBuild.push_back(bo);
                             element.storePaths.insert(bo.path);
                         },
-                        [&](BuildableFromDrv bfd) {
+                        [&](DerivedPathWithHintsBuilt bfd) {
                             // TODO: Why are we querying if we know the output
                             // names already? Is it just to figure out what the
                             // default one is?
                             for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
-                                pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}});
+                                pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}});
                                 element.storePaths.insert(output.second);
                             }
                         },
@@ -391,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
         auto matchers = getMatchers(store);
 
         // FIXME: code duplication
-        std::vector<BuildableReq> pathsToBuild;
+        std::vector<DerivedPath> pathsToBuild;
 
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
@@ -426,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                     attrPath,
                 };
 
-                pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME
+                pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME
             }
         }
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 2e9bb41cc..ba60e57d8 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon
 
         auto app = installable->toApp(*state);
 
-        state->store->buildPaths(toBuildableReqs(app.context));
+        state->store->buildPaths(toDerivedPaths(app.context));
 
         Strings allArgs{app.program};
         for (auto & i : args) allArgs.push_back(i);

From 179582872de60863fcabcf471f98930a25fd6df3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 5 Apr 2021 10:05:21 -0400
Subject: [PATCH 845/882] Make `DerivedPathWithHints` a newtype

This allows us to namespace its constructors under it.
---
 src/libcmd/command.cc      |  6 +++---
 src/libcmd/installables.cc | 26 +++++++++++++-------------
 src/libstore/buildable.cc  |  4 ++--
 src/libstore/buildable.hh  | 15 ++++++++++++++-
 src/nix/build.cc           |  6 +++---
 src/nix/develop.cc         |  6 +++---
 src/nix/log.cc             |  6 +++---
 src/nix/profile.cc         |  6 +++---
 8 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index dc1fbc43f..9da470c15 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -170,10 +170,10 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](DerivedPathOpaque bo) {
+            [&](DerivedPathWithHints::Opaque bo) {
                 result.push_back(bo.path);
             },
-            [&](DerivedPathWithHintsBuilt bfd) {
+            [&](DerivedPathWithHints::Built bfd) {
                 for (auto & output : bfd.outputs) {
                     /* Output path should be known because we just tried to
                        build it. */
@@ -181,7 +181,7 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
                     result.push_back(*output.second);
                 }
             },
-        }, buildable);
+        }, buildable.raw());
     }
 
     if (result.size() != 1)
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index f091ac186..5d3026c1a 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -329,14 +329,14 @@ struct InstallableStorePath : Installable
             for (auto & [name, output] : drv.outputsAndOptPaths(*store))
                 outputs.emplace(name, output.second);
             return {
-                DerivedPathWithHintsBuilt {
+                DerivedPathWithHints::Built {
                     .drvPath = storePath,
                     .outputs = std::move(outputs)
                 }
             };
         } else {
             return {
-                DerivedPathOpaque {
+                DerivedPathWithHints::Opaque {
                     .path = storePath,
                 }
             };
@@ -364,7 +364,7 @@ DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints()
     }
 
     for (auto & i : drvsToOutputs)
-        res.push_back(DerivedPathWithHintsBuilt { i.first, i.second });
+        res.push_back(DerivedPathWithHints::Built { i.first, i.second });
 
     return res;
 }
@@ -684,17 +684,17 @@ DerivedPathsWithHints build(ref<Store> store, Realise mode,
     for (auto & i : installables) {
         for (auto & b : i->toDerivedPathsWithHints()) {
             std::visit(overloaded {
-                [&](DerivedPathOpaque bo) {
+                [&](DerivedPathWithHints::Opaque bo) {
                     pathsToBuild.push_back(bo);
                 },
-                [&](DerivedPathWithHintsBuilt bfd) {
+                [&](DerivedPathWithHints::Built bfd) {
                     StringSet outputNames;
                     for (auto & output : bfd.outputs)
                         outputNames.insert(output.first);
                     pathsToBuild.push_back(
                         DerivedPath::Built{bfd.drvPath, outputNames});
                 },
-            }, b);
+            }, b.raw());
             buildables.push_back(std::move(b));
         }
     }
@@ -717,10 +717,10 @@ std::set<RealisedPath> toRealisedPaths(
     if (operateOn == OperateOn::Output) {
         for (auto & b : build(store, mode, installables))
             std::visit(overloaded {
-                [&](DerivedPathOpaque bo) {
+                [&](DerivedPathWithHints::Opaque bo) {
                     res.insert(bo.path);
                 },
-                [&](DerivedPathWithHintsBuilt bfd) {
+                [&](DerivedPathWithHints::Built bfd) {
                     auto drv = store->readDerivation(bfd.drvPath);
                     auto outputHashes = staticOutputHashes(*store, drv);
                     for (auto & output : bfd.outputs) {
@@ -745,14 +745,14 @@ std::set<RealisedPath> toRealisedPaths(
                         }
                     }
                 },
-            }, b);
+            }, b.raw());
     } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
         for (auto & i : installables)
             for (auto & b : i->toDerivedPathsWithHints())
-                if (auto bfd = std::get_if<DerivedPathWithHintsBuilt>(&b))
+                if (auto bfd = std::get_if<DerivedPathWithHints::Built>(&b))
                     res.insert(bfd->drvPath);
     }
 
@@ -789,7 +789,7 @@ StorePathSet toDerivations(ref<Store> store,
     for (auto & i : installables)
         for (auto & b : i->toDerivedPathsWithHints())
             std::visit(overloaded {
-                [&](DerivedPathOpaque bo) {
+                [&](DerivedPathWithHints::Opaque bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
                     auto derivers = store->queryValidDerivers(bo.path);
@@ -798,10 +798,10 @@ StorePathSet toDerivations(ref<Store> store,
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
                 },
-                [&](DerivedPathWithHintsBuilt bfd) {
+                [&](DerivedPathWithHints::Built bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
-            }, b);
+            }, b.raw());
 
     return drvPaths;
 }
diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc
index a8c0c70b1..eee38ba10 100644
--- a/src/libstore/buildable.cc
+++ b/src/libstore/buildable.cc
@@ -11,7 +11,7 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
     return res;
 }
 
-nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref<Store> store) const {
+nlohmann::json DerivedPathWithHints::Built::toJSON(ref<Store> store) const {
     nlohmann::json res;
     res["drvPath"] = store->printStorePath(drvPath);
     for (const auto& [output, path] : outputs) {
@@ -25,7 +25,7 @@ nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildab
     for (const DerivedPathWithHints & buildable : buildables) {
         std::visit([&res, store](const auto & buildable) {
             res.push_back(buildable.toJSON(store));
-        }, buildable);
+        }, buildable.raw());
     }
     return res;
 }
diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh
index 0a0cf8105..ce5ae5fc0 100644
--- a/src/libstore/buildable.hh
+++ b/src/libstore/buildable.hh
@@ -56,11 +56,24 @@ struct DerivedPathWithHintsBuilt {
     static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view);
 };
 
-using DerivedPathWithHints = std::variant<
+using _DerivedPathWithHintsRaw = std::variant<
     DerivedPath::Opaque,
     DerivedPathWithHintsBuilt
 >;
 
+struct DerivedPathWithHints : _DerivedPathWithHintsRaw {
+    using Raw = _DerivedPathWithHintsRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = DerivedPathWithHintsBuilt;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+};
+
 typedef std::vector<DerivedPathWithHints> DerivedPathsWithHints;
 
 nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 0529ed382..03159b6cc 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                 for (const auto & [_i, buildable] : enumerate(buildables)) {
                     auto i = _i;
                     std::visit(overloaded {
-                        [&](DerivedPathOpaque bo) {
+                        [&](DerivedPathWithHints::Opaque bo) {
                             std::string symlink = outLink;
                             if (i) symlink += fmt("-%d", i);
                             store2->addPermRoot(bo.path, absPath(symlink));
                         },
-                        [&](DerivedPathWithHintsBuilt bfd) {
+                        [&](DerivedPathWithHints::Built bfd) {
                             auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
                             for (auto & output : builtOutputs) {
                                 std::string symlink = outLink;
@@ -75,7 +75,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                                 store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
-                    }, buildable);
+                    }, buildable.raw());
                 }
 
         updateProfile(buildables);
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index cae6ded40..7cc7b85be 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -277,14 +277,14 @@ struct Common : InstallableCommand, MixProfile
                 }
             };
             std::visit(overloaded {
-                [&](const DerivedPathOpaque & bo) {
+                [&](const DerivedPathWithHints::Opaque & bo) {
                     doRedirect(bo.path);
                 },
-                [&](const DerivedPathWithHintsBuilt & bfd) {
+                [&](const DerivedPathWithHints::Built & bfd) {
                     for (auto & [outputName, path] : bfd.outputs)
                         if (path) doRedirect(*path);
                 },
-            }, buildable);
+            }, buildable.raw());
         }
 
         return rewriteStrings(script, rewrites);
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 5010e3326..638bb5073 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -35,13 +35,13 @@ struct CmdLog : InstallableCommand
         RunPager pager;
         for (auto & sub : subs) {
             auto log = std::visit(overloaded {
-                [&](DerivedPathOpaque bo) {
+                [&](DerivedPathWithHints::Opaque bo) {
                     return sub->getBuildLog(bo.path);
                 },
-                [&](DerivedPathWithHintsBuilt bfd) {
+                [&](DerivedPathWithHints::Built bfd) {
                     return sub->getBuildLog(bfd.drvPath);
                 },
-            }, b);
+            }, b.raw());
             if (!log) continue;
             stopProgressBar();
             printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index ad824dd70..667904cd2 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -259,11 +259,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     ProfileElement element;
 
                     std::visit(overloaded {
-                        [&](DerivedPathOpaque bo) {
+                        [&](DerivedPathWithHints::Opaque bo) {
                             pathsToBuild.push_back(bo);
                             element.storePaths.insert(bo.path);
                         },
-                        [&](DerivedPathWithHintsBuilt bfd) {
+                        [&](DerivedPathWithHints::Built bfd) {
                             // TODO: Why are we querying if we know the output
                             // names already? Is it just to figure out what the
                             // default one is?
@@ -272,7 +272,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                                 element.storePaths.insert(output.second);
                             }
                         },
-                    }, buildable);
+                    }, buildable.raw());
 
                     manifest.elements.emplace_back(std::move(element));
                 }

From d8fa7517fad4272e20ff9b9b740c91158bc685e2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 5 Apr 2021 10:33:28 -0400
Subject: [PATCH 846/882] buildable.{cc,hh} -> derived-path.{cc,hh}

---
 src/libcmd/installables.hh                     | 2 +-
 src/libmain/shared.hh                          | 2 +-
 src/libstore/{buildable.cc => derived-path.cc} | 2 +-
 src/libstore/{buildable.hh => derived-path.hh} | 0
 src/libstore/path-with-outputs.hh              | 2 +-
 src/libstore/store-api.hh                      | 2 +-
 6 files changed, 5 insertions(+), 5 deletions(-)
 rename src/libstore/{buildable.cc => derived-path.cc} (98%)
 rename src/libstore/{buildable.hh => derived-path.hh} (100%)

diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index 0bc932b52..403403c07 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -3,7 +3,7 @@
 #include "util.hh"
 #include "path.hh"
 #include "path-with-outputs.hh"
-#include "buildable.hh"
+#include "derived-path.hh"
 #include "eval.hh"
 #include "flake/flake.hh"
 
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 9cb9e6da2..05277d90a 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -4,7 +4,7 @@
 #include "args.hh"
 #include "common-args.hh"
 #include "path.hh"
-#include "buildable.hh"
+#include "derived-path.hh"
 
 #include <signal.h>
 
diff --git a/src/libstore/buildable.cc b/src/libstore/derived-path.cc
similarity index 98%
rename from src/libstore/buildable.cc
rename to src/libstore/derived-path.cc
index eee38ba10..13833c58e 100644
--- a/src/libstore/buildable.cc
+++ b/src/libstore/derived-path.cc
@@ -1,4 +1,4 @@
-#include "buildable.hh"
+#include "derived-path.hh"
 #include "store-api.hh"
 
 #include <nlohmann/json.hpp>
diff --git a/src/libstore/buildable.hh b/src/libstore/derived-path.hh
similarity index 100%
rename from src/libstore/buildable.hh
rename to src/libstore/derived-path.hh
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
index 749348398..4c4023dcb 100644
--- a/src/libstore/path-with-outputs.hh
+++ b/src/libstore/path-with-outputs.hh
@@ -3,7 +3,7 @@
 #include <variant>
 
 #include "path.hh"
-#include "buildable.hh"
+#include "derived-path.hh"
 
 namespace nix {
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 483f3c5fa..f66298991 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -2,7 +2,7 @@
 
 #include "realisation.hh"
 #include "path.hh"
-#include "buildable.hh"
+#include "derived-path.hh"
 #include "hash.hh"
 #include "content-address.hh"
 #include "serialise.hh"

From 125a824228dbac0bb82023953f45318ea93e7ffa Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 5 Apr 2021 10:56:48 -0400
Subject: [PATCH 847/882] Document the derived path types.

---
 src/libstore/derived-path.hh | 50 +++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index ce5ae5fc0..7a2fe59de 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -2,7 +2,6 @@
 
 #include "util.hh"
 #include "path.hh"
-#include "path.hh"
 
 #include <optional>
 
@@ -12,6 +11,13 @@ namespace nix {
 
 class Store;
 
+/**
+ * An opaque derived path.
+ *
+ * Opaque derived paths are just store paths, and fully evaluated. They
+ * cannot be simplified further. Since they are opaque, they cannot be
+ * built, but they can fetched.
+ */
 struct DerivedPathOpaque {
     StorePath path;
 
@@ -20,6 +26,18 @@ struct DerivedPathOpaque {
     static DerivedPathOpaque parse(const Store & store, std::string_view);
 };
 
+/**
+ * A derived path that is built from a derivation
+ *
+ * Built derived paths are pair of a derivation and some output names.
+ * They are evaluated by building the derivation, and then replacing the
+ * output names with the resulting outputs.
+ *
+ * Note that does mean a derived store paths evaluates to multiple
+ * opaque paths, which is sort of icky as expressions are supposed to
+ * evaluate to single values. Perhaps this should have just a single
+ * output name.
+ */
 struct DerivedPathBuilt {
     StorePath drvPath;
     std::set<std::string> outputs;
@@ -33,6 +51,16 @@ using _DerivedPathRaw = std::variant<
     DerivedPathBuilt
 >;
 
+/**
+ * A "derived path" is a very simple sort of expression that evaluates
+ * to (concrete) store path. It is either:
+ *
+ * - opaque, in which case it is just a concrete store path with
+ *   possibly no known derivation
+ *
+ * - built, in which case it is a pair of a derivation path and an
+ *   output name.
+ */
 struct DerivedPath : _DerivedPathRaw {
     using Raw = _DerivedPathRaw;
     using Raw::Raw;
@@ -48,6 +76,11 @@ struct DerivedPath : _DerivedPathRaw {
     static DerivedPath parse(const Store & store, std::string_view);
 };
 
+/**
+ * A built derived path with hints in the form of optional concrete output paths.
+ *
+ * See 'DerivedPathWithHints' for more an explanation.
+ */
 struct DerivedPathWithHintsBuilt {
     StorePath drvPath;
     std::map<std::string, std::optional<StorePath>> outputs;
@@ -61,6 +94,21 @@ using _DerivedPathWithHintsRaw = std::variant<
     DerivedPathWithHintsBuilt
 >;
 
+/**
+ * A derived path with hints in the form of optional concrete output paths in the built case.
+ *
+ * This type is currently just used by the CLI. The paths are filled in
+ * during evaluation for derivations that know what paths they will
+ * produce in advanced, i.e. input-addressed or fixed-output content
+ * addressed derivations.
+ *
+ * That isn't very good, because it puts floating content-addressed
+ * derivations "at a disadvantage". It would be better to never rely on
+ * the output path of unbuilt derivations, and exclusively use the
+ * realizations types to work with built derivations' concrete output
+ * paths.
+ */
+// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd.
 struct DerivedPathWithHints : _DerivedPathWithHintsRaw {
     using Raw = _DerivedPathWithHintsRaw;
     using Raw::Raw;

From 9f28dd97ae6afc68f0574a251325336c12d60c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gohla?= <b.gohla@gmx.de>
Date: Mon, 5 Apr 2021 21:24:55 +0100
Subject: [PATCH 848/882] Revert "Use upstream nlohmann_json"

This reverts commit 4145cd2da002e1bd8affa0392c80118eabe58e3c.
---
 src/nlohmann/json.hpp | 20406 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 20406 insertions(+)
 create mode 100644 src/nlohmann/json.hpp

diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
new file mode 100644
index 000000000..c9af0bed3
--- /dev/null
+++ b/src/nlohmann/json.hpp
@@ -0,0 +1,20406 @@
+/*
+    __ _____ _____ _____
+ __|  |   __|     |   | |  JSON for Modern C++
+|  |  |__   |  |  | | | |  version 3.5.0
+|_____|_____|_____|_|___|  https://github.com/nlohmann/json
+
+Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+SPDX-License-Identifier: MIT
+Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>.
+
+Permission is hereby  granted, free of charge, to any  person obtaining a copy
+of this software and associated  documentation files (the "Software"), to deal
+in the Software  without restriction, including without  limitation the rights
+to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
+copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
+IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
+FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
+AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
+LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef NLOHMANN_JSON_HPP
+#define NLOHMANN_JSON_HPP
+
+#define NLOHMANN_JSON_VERSION_MAJOR 3
+#define NLOHMANN_JSON_VERSION_MINOR 5
+#define NLOHMANN_JSON_VERSION_PATCH 0
+
+#include <algorithm> // all_of, find, for_each
+#include <cassert> // assert
+#include <ciso646> // and, not, or
+#include <cstddef> // nullptr_t, ptrdiff_t, size_t
+#include <functional> // hash, less
+#include <initializer_list> // initializer_list
+#include <iosfwd> // istream, ostream
+#include <iterator> // random_access_iterator_tag
+#include <numeric> // accumulate
+#include <string> // string, stoi, to_string
+#include <utility> // declval, forward, move, pair, swap
+
+// #include <nlohmann/json_fwd.hpp>
+#ifndef NLOHMANN_JSON_FWD_HPP
+#define NLOHMANN_JSON_FWD_HPP
+
+#include <cstdint> // int64_t, uint64_t
+#include <map> // map
+#include <memory> // allocator
+#include <string> // string
+#include <vector> // vector
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename T = void, typename SFINAE = void>
+struct adl_serializer;
+
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer>
+class basic_json;
+
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+template<typename BasicJsonType>
+class json_pointer;
+
+/*!
+@brief default JSON class
+
+This type is the default specialization of the @ref basic_json class which
+uses the standard template types.
+
+@since version 1.0.0
+*/
+using json = basic_json<>;
+}  // namespace nlohmann
+
+#endif
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+// This file contains all internal macro definitions
+// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them
+
+// exclude unsupported compilers
+#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK)
+    #if defined(__clang__)
+        #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
+            #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
+        #endif
+    #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
+        #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800
+            #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+        #endif
+    #endif
+#endif
+
+// disable float-equal warnings on GCC/clang
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+
+// disable documentation warnings on clang
+#if defined(__clang__)
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wdocumentation"
+#endif
+
+// allow for portable deprecation warnings
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #define JSON_DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+    #define JSON_DEPRECATED __declspec(deprecated)
+#else
+    #define JSON_DEPRECATED
+#endif
+
+// allow to disable exceptions
+#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
+    #define JSON_THROW(exception) throw exception
+    #define JSON_TRY try
+    #define JSON_CATCH(exception) catch(exception)
+    #define JSON_INTERNAL_CATCH(exception) catch(exception)
+#else
+    #define JSON_THROW(exception) std::abort()
+    #define JSON_TRY if(true)
+    #define JSON_CATCH(exception) if(false)
+    #define JSON_INTERNAL_CATCH(exception) if(false)
+#endif
+
+// override exception macros
+#if defined(JSON_THROW_USER)
+    #undef JSON_THROW
+    #define JSON_THROW JSON_THROW_USER
+#endif
+#if defined(JSON_TRY_USER)
+    #undef JSON_TRY
+    #define JSON_TRY JSON_TRY_USER
+#endif
+#if defined(JSON_CATCH_USER)
+    #undef JSON_CATCH
+    #define JSON_CATCH JSON_CATCH_USER
+    #undef JSON_INTERNAL_CATCH
+    #define JSON_INTERNAL_CATCH JSON_CATCH_USER
+#endif
+#if defined(JSON_INTERNAL_CATCH_USER)
+    #undef JSON_INTERNAL_CATCH
+    #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER
+#endif
+
+// manual branch prediction
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #define JSON_LIKELY(x)      __builtin_expect(!!(x), 1)
+    #define JSON_UNLIKELY(x)    __builtin_expect(!!(x), 0)
+#else
+    #define JSON_LIKELY(x)      x
+    #define JSON_UNLIKELY(x)    x
+#endif
+
+// C++ language standard detection
+#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464
+    #define JSON_HAS_CPP_17
+    #define JSON_HAS_CPP_14
+#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1)
+    #define JSON_HAS_CPP_14
+#endif
+
+/*!
+@brief macro to briefly define a mapping between an enum and JSON
+@def NLOHMANN_JSON_SERIALIZE_ENUM
+@since version 3.4.0
+*/
+#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...)                                           \
+    template<typename BasicJsonType>                                                           \
+    inline void to_json(BasicJsonType& j, const ENUM_TYPE& e)                                  \
+    {                                                                                          \
+        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
+        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
+        auto it = std::find_if(std::begin(m), std::end(m),                                     \
+                               [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+        {                                                                                      \
+            return ej_pair.first == e;                                                         \
+        });                                                                                    \
+        j = ((it != std::end(m)) ? it : std::begin(m))->second;                                \
+    }                                                                                          \
+    template<typename BasicJsonType>                                                           \
+    inline void from_json(const BasicJsonType& j, ENUM_TYPE& e)                                \
+    {                                                                                          \
+        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
+        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
+        auto it = std::find_if(std::begin(m), std::end(m),                                     \
+                               [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+        {                                                                                      \
+            return ej_pair.second == j;                                                        \
+        });                                                                                    \
+        e = ((it != std::end(m)) ? it : std::begin(m))->first;                                 \
+    }
+
+// Ugly macros to avoid uglier copy-paste when specializing basic_json. They
+// may be removed in the future once the class is split.
+
+#define NLOHMANN_BASIC_JSON_TPL_DECLARATION                                \
+    template<template<typename, typename, typename...> class ObjectType,   \
+             template<typename, typename...> class ArrayType,              \
+             class StringType, class BooleanType, class NumberIntegerType, \
+             class NumberUnsignedType, class NumberFloatType,              \
+             template<typename> class AllocatorType,                       \
+             template<typename, typename = void> class JSONSerializer>
+
+#define NLOHMANN_BASIC_JSON_TPL                                            \
+    basic_json<ObjectType, ArrayType, StringType, BooleanType,             \
+    NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
+    AllocatorType, JSONSerializer>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+
+#include <ciso646> // not
+#include <cstddef> // size_t
+#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
+
+namespace nlohmann
+{
+namespace detail
+{
+// alias templates to reduce boilerplate
+template<bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+template<typename T>
+using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
+// implementation of C++14 index_sequence and affiliates
+// source: https://stackoverflow.com/a/32223343
+template<std::size_t... Ints>
+struct index_sequence
+{
+    using type = index_sequence;
+    using value_type = std::size_t;
+    static constexpr std::size_t size() noexcept
+    {
+        return sizeof...(Ints);
+    }
+};
+
+template<class Sequence1, class Sequence2>
+struct merge_and_renumber;
+
+template<std::size_t... I1, std::size_t... I2>
+struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
+        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
+
+template<std::size_t N>
+struct make_index_sequence
+    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
+      typename make_index_sequence < N - N / 2 >::type > {};
+
+template<> struct make_index_sequence<0> : index_sequence<> {};
+template<> struct make_index_sequence<1> : index_sequence<0> {};
+
+template<typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+
+// dispatch utility (taken from ranges-v3)
+template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
+template<> struct priority_tag<0> {};
+
+// taken from ranges-v3
+template<typename T>
+struct static_const
+{
+    static constexpr T value{};
+};
+
+template<typename T>
+constexpr T static_const<T>::value;
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+#include <ciso646> // not
+#include <limits> // numeric_limits
+#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type
+#include <utility> // declval
+
+// #include <nlohmann/json_fwd.hpp>
+
+// #include <nlohmann/detail/iterators/iterator_traits.hpp>
+
+
+#include <iterator> // random_access_iterator_tag
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename ...Ts> struct make_void
+{
+    using type = void;
+};
+template <typename ...Ts> using void_t = typename make_void<Ts...>::type;
+} // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename It, typename = void>
+struct iterator_types {};
+
+template <typename It>
+struct iterator_types <
+    It,
+    void_t<typename It::difference_type, typename It::value_type, typename It::pointer,
+    typename It::reference, typename It::iterator_category >>
+{
+    using difference_type = typename It::difference_type;
+    using value_type = typename It::value_type;
+    using pointer = typename It::pointer;
+    using reference = typename It::reference;
+    using iterator_category = typename It::iterator_category;
+};
+
+// This is required as some compilers implement std::iterator_traits in a way that
+// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341.
+template <typename T, typename = void>
+struct iterator_traits
+{
+};
+
+template <typename T>
+struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >>
+            : iterator_types<T>
+{
+};
+
+template <typename T>
+struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>>
+{
+    using iterator_category = std::random_access_iterator_tag;
+    using value_type = T;
+    using difference_type = ptrdiff_t;
+    using pointer = T*;
+    using reference = T&;
+};
+}
+}
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+
+#include <type_traits>
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
+
+// http://en.cppreference.com/w/cpp/experimental/is_detected
+namespace nlohmann
+{
+namespace detail
+{
+struct nonesuch
+{
+    nonesuch() = delete;
+    ~nonesuch() = delete;
+    nonesuch(nonesuch const&) = delete;
+    void operator=(nonesuch const&) = delete;
+};
+
+template <class Default,
+          class AlwaysVoid,
+          template <class...> class Op,
+          class... Args>
+struct detector
+{
+    using value_t = std::false_type;
+    using type = Default;
+};
+
+template <class Default, template <class...> class Op, class... Args>
+struct detector<Default, void_t<Op<Args...>>, Op, Args...>
+{
+    using value_t = std::true_type;
+    using type = Op<Args...>;
+};
+
+template <template <class...> class Op, class... Args>
+using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t;
+
+template <template <class...> class Op, class... Args>
+using detected_t = typename detector<nonesuch, void, Op, Args...>::type;
+
+template <class Default, template <class...> class Op, class... Args>
+using detected_or = detector<Default, void, Op, Args...>;
+
+template <class Default, template <class...> class Op, class... Args>
+using detected_or_t = typename detected_or<Default, Op, Args...>::type;
+
+template <class Expected, template <class...> class Op, class... Args>
+using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>;
+
+template <class To, template <class...> class Op, class... Args>
+using is_detected_convertible =
+    std::is_convertible<detected_t<Op, Args...>, To>;
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+namespace nlohmann
+{
+/*!
+@brief detail namespace with internal helper functions
+
+This namespace collects functions that should not be exposed,
+implementations of some @ref basic_json methods, and meta-programming helpers.
+
+@since version 2.1.0
+*/
+namespace detail
+{
+/////////////
+// helpers //
+/////////////
+
+// Note to maintainers:
+//
+// Every trait in this file expects a non CV-qualified type.
+// The only exceptions are in the 'aliases for detected' section
+// (i.e. those of the form: decltype(T::member_function(std::declval<T>())))
+//
+// In this case, T has to be properly CV-qualified to constraint the function arguments
+// (e.g. to_json(BasicJsonType&, const T&))
+
+template<typename> struct is_basic_json : std::false_type {};
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+
+//////////////////////////
+// aliases for detected //
+//////////////////////////
+
+template <typename T>
+using mapped_type_t = typename T::mapped_type;
+
+template <typename T>
+using key_type_t = typename T::key_type;
+
+template <typename T>
+using value_type_t = typename T::value_type;
+
+template <typename T>
+using difference_type_t = typename T::difference_type;
+
+template <typename T>
+using pointer_t = typename T::pointer;
+
+template <typename T>
+using reference_t = typename T::reference;
+
+template <typename T>
+using iterator_category_t = typename T::iterator_category;
+
+template <typename T>
+using iterator_t = typename T::iterator;
+
+template <typename T, typename... Args>
+using to_json_function = decltype(T::to_json(std::declval<Args>()...));
+
+template <typename T, typename... Args>
+using from_json_function = decltype(T::from_json(std::declval<Args>()...));
+
+template <typename T, typename U>
+using get_template_function = decltype(std::declval<T>().template get<U>());
+
+// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
+template <typename BasicJsonType, typename T, typename = void>
+struct has_from_json : std::false_type {};
+
+template <typename BasicJsonType, typename T>
+struct has_from_json<BasicJsonType, T,
+           enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+    static constexpr bool value =
+        is_detected_exact<void, from_json_function, serializer,
+        const BasicJsonType&, T&>::value;
+};
+
+// This trait checks if JSONSerializer<T>::from_json(json const&) exists
+// this overload is used for non-default-constructible user-defined-types
+template <typename BasicJsonType, typename T, typename = void>
+struct has_non_default_from_json : std::false_type {};
+
+template<typename BasicJsonType, typename T>
+struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+    static constexpr bool value =
+        is_detected_exact<T, from_json_function, serializer,
+        const BasicJsonType&>::value;
+};
+
+// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
+// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion.
+template <typename BasicJsonType, typename T, typename = void>
+struct has_to_json : std::false_type {};
+
+template <typename BasicJsonType, typename T>
+struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+    static constexpr bool value =
+        is_detected_exact<void, to_json_function, serializer, BasicJsonType&,
+        T>::value;
+};
+
+
+///////////////////
+// is_ functions //
+///////////////////
+
+template <typename T, typename = void>
+struct is_iterator_traits : std::false_type {};
+
+template <typename T>
+struct is_iterator_traits<iterator_traits<T>>
+{
+  private:
+    using traits = iterator_traits<T>;
+
+  public:
+    static constexpr auto value =
+        is_detected<value_type_t, traits>::value &&
+        is_detected<difference_type_t, traits>::value &&
+        is_detected<pointer_t, traits>::value &&
+        is_detected<iterator_category_t, traits>::value &&
+        is_detected<reference_t, traits>::value;
+};
+
+// source: https://stackoverflow.com/a/37193089/4116453
+
+template <typename T, typename = void>
+struct is_complete_type : std::false_type {};
+
+template <typename T>
+struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {};
+
+template <typename BasicJsonType, typename CompatibleObjectType,
+          typename = void>
+struct is_compatible_object_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type_impl <
+    BasicJsonType, CompatibleObjectType,
+    enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and
+    is_detected<key_type_t, CompatibleObjectType>::value >>
+{
+
+    using object_t = typename BasicJsonType::object_t;
+
+    // macOS's is_constructible does not play well with nonesuch...
+    static constexpr bool value =
+        std::is_constructible<typename object_t::key_type,
+        typename CompatibleObjectType::key_type>::value and
+        std::is_constructible<typename object_t::mapped_type,
+        typename CompatibleObjectType::mapped_type>::value;
+};
+
+template <typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type
+    : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {};
+
+template <typename BasicJsonType, typename ConstructibleObjectType,
+          typename = void>
+struct is_constructible_object_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type_impl <
+    BasicJsonType, ConstructibleObjectType,
+    enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and
+    is_detected<key_type_t, ConstructibleObjectType>::value >>
+{
+    using object_t = typename BasicJsonType::object_t;
+
+    static constexpr bool value =
+        (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and
+         std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or
+        (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or
+         has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value);
+};
+
+template <typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type
+    : is_constructible_object_type_impl<BasicJsonType,
+      ConstructibleObjectType> {};
+
+template <typename BasicJsonType, typename CompatibleStringType,
+          typename = void>
+struct is_compatible_string_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleStringType>
+struct is_compatible_string_type_impl <
+    BasicJsonType, CompatibleStringType,
+    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
+    value_type_t, CompatibleStringType>::value >>
+{
+    static constexpr auto value =
+        std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value;
+};
+
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_compatible_string_type
+    : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
+
+template <typename BasicJsonType, typename ConstructibleStringType,
+          typename = void>
+struct is_constructible_string_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type_impl <
+    BasicJsonType, ConstructibleStringType,
+    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
+    value_type_t, ConstructibleStringType>::value >>
+{
+    static constexpr auto value =
+        std::is_constructible<ConstructibleStringType,
+        typename BasicJsonType::string_t>::value;
+};
+
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type
+    : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
+
+template <typename BasicJsonType, typename CompatibleArrayType, typename = void>
+struct is_compatible_array_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type_impl <
+    BasicJsonType, CompatibleArrayType,
+    enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and
+    is_detected<iterator_t, CompatibleArrayType>::value and
+// This is needed because json_reverse_iterator has a ::iterator type...
+// Therefore it is detected as a CompatibleArrayType.
+// The real fix would be to have an Iterable concept.
+    not is_iterator_traits<
+    iterator_traits<CompatibleArrayType>>::value >>
+{
+    static constexpr bool value =
+        std::is_constructible<BasicJsonType,
+        typename CompatibleArrayType::value_type>::value;
+};
+
+template <typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type
+    : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType, typename = void>
+struct is_constructible_array_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+    BasicJsonType, ConstructibleArrayType,
+    enable_if_t<std::is_same<ConstructibleArrayType,
+    typename BasicJsonType::value_type>::value >>
+            : std::true_type {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+    BasicJsonType, ConstructibleArrayType,
+    enable_if_t<not std::is_same<ConstructibleArrayType,
+    typename BasicJsonType::value_type>::value and
+    is_detected<value_type_t, ConstructibleArrayType>::value and
+    is_detected<iterator_t, ConstructibleArrayType>::value and
+    is_complete_type<
+    detected_t<value_type_t, ConstructibleArrayType>>::value >>
+{
+    static constexpr bool value =
+        // This is needed because json_reverse_iterator has a ::iterator type,
+        // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`...
+        // Therefore it is detected as a ConstructibleArrayType.
+        // The real fix would be to have an Iterable concept.
+        not is_iterator_traits <
+        iterator_traits<ConstructibleArrayType >>::value and
+
+        (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or
+         has_from_json<BasicJsonType,
+         typename ConstructibleArrayType::value_type>::value or
+         has_non_default_from_json <
+         BasicJsonType, typename ConstructibleArrayType::value_type >::value);
+};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type
+    : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {};
+
+template <typename RealIntegerType, typename CompatibleNumberIntegerType,
+          typename = void>
+struct is_compatible_integer_type_impl : std::false_type {};
+
+template <typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type_impl <
+    RealIntegerType, CompatibleNumberIntegerType,
+    enable_if_t<std::is_integral<RealIntegerType>::value and
+    std::is_integral<CompatibleNumberIntegerType>::value and
+    not std::is_same<bool, CompatibleNumberIntegerType>::value >>
+{
+    // is there an assert somewhere on overflows?
+    using RealLimits = std::numeric_limits<RealIntegerType>;
+    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
+
+    static constexpr auto value =
+        std::is_constructible<RealIntegerType,
+        CompatibleNumberIntegerType>::value and
+        CompatibleLimits::is_integer and
+        RealLimits::is_signed == CompatibleLimits::is_signed;
+};
+
+template <typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type
+    : is_compatible_integer_type_impl<RealIntegerType,
+      CompatibleNumberIntegerType> {};
+
+template <typename BasicJsonType, typename CompatibleType, typename = void>
+struct is_compatible_type_impl: std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type_impl <
+    BasicJsonType, CompatibleType,
+    enable_if_t<is_complete_type<CompatibleType>::value >>
+{
+    static constexpr bool value =
+        has_to_json<BasicJsonType, CompatibleType>::value;
+};
+
+template <typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type
+    : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+
+#include <exception> // exception
+#include <stdexcept> // runtime_error
+#include <string> // to_string
+
+// #include <nlohmann/detail/input/position_t.hpp>
+
+
+#include <cstddef> // size_t
+
+namespace nlohmann
+{
+namespace detail
+{
+/// struct to capture the start position of the current token
+struct position_t
+{
+    /// the total number of characters read
+    std::size_t chars_read_total = 0;
+    /// the number of characters read in the current line
+    std::size_t chars_read_current_line = 0;
+    /// the number of lines read
+    std::size_t lines_read = 0;
+
+    /// conversion to size_t to preserve SAX interface
+    constexpr operator size_t() const
+    {
+        return chars_read_total;
+    }
+};
+
+}
+}
+
+
+namespace nlohmann
+{
+namespace detail
+{
+////////////////
+// exceptions //
+////////////////
+
+/*!
+@brief general exception of the @ref basic_json class
+
+This class is an extension of `std::exception` objects with a member @a id for
+exception ids. It is used as the base class for all exceptions thrown by the
+@ref basic_json class. This class can hence be used as "wildcard" to catch
+exceptions.
+
+Subclasses:
+- @ref parse_error for exceptions indicating a parse error
+- @ref invalid_iterator for exceptions indicating errors with iterators
+- @ref type_error for exceptions indicating executing a member function with
+                  a wrong type
+- @ref out_of_range for exceptions indicating access out of the defined range
+- @ref other_error for exceptions indicating other library errors
+
+@internal
+@note To have nothrow-copy-constructible exceptions, we internally use
+      `std::runtime_error` which can cope with arbitrary-length error messages.
+      Intermediate strings are built with static functions and then passed to
+      the actual constructor.
+@endinternal
+
+@liveexample{The following code shows how arbitrary library exceptions can be
+caught.,exception}
+
+@since version 3.0.0
+*/
+class exception : public std::exception
+{
+  public:
+    /// returns the explanatory string
+    const char* what() const noexcept override
+    {
+        return m.what();
+    }
+
+    /// the id of the exception
+    const int id;
+
+  protected:
+    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {}
+
+    static std::string name(const std::string& ename, int id_)
+    {
+        return "[json.exception." + ename + "." + std::to_string(id_) + "] ";
+    }
+
+  private:
+    /// an exception object as storage for error messages
+    std::runtime_error m;
+};
+
+/*!
+@brief exception indicating a parse error
+
+This exception is thrown by the library when a parse error occurs. Parse errors
+can occur during the deserialization of JSON text, CBOR, MessagePack, as well
+as when using JSON Patch.
+
+Member @a byte holds the byte index of the last read character in the input
+file.
+
+Exceptions have ids 1xx.
+
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
+json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
+json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
+json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
+json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
+json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`.
+json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
+json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
+json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
+json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
+json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
+json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
+json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet).
+
+@note For an input with n bytes, 1 is the index of the first character and n+1
+      is the index of the terminating null byte or the end of file. This also
+      holds true when reading a byte vector (CBOR or MessagePack).
+
+@liveexample{The following code shows how a `parse_error` exception can be
+caught.,parse_error}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class parse_error : public exception
+{
+  public:
+    /*!
+    @brief create a parse error exception
+    @param[in] id_       the id of the exception
+    @param[in] position  the position where the error occurred (or with
+                         chars_read_total=0 if the position cannot be
+                         determined)
+    @param[in] what_arg  the explanatory string
+    @return parse_error object
+    */
+    static parse_error create(int id_, const position_t& pos, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id_) + "parse error" +
+                        position_string(pos) + ": " + what_arg;
+        return parse_error(id_, pos.chars_read_total, w.c_str());
+    }
+
+    static parse_error create(int id_, std::size_t byte_, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id_) + "parse error" +
+                        (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") +
+                        ": " + what_arg;
+        return parse_error(id_, byte_, w.c_str());
+    }
+
+    /*!
+    @brief byte index of the parse error
+
+    The byte index of the last read character in the input file.
+
+    @note For an input with n bytes, 1 is the index of the first character and
+          n+1 is the index of the terminating null byte or the end of file.
+          This also holds true when reading a byte vector (CBOR or MessagePack).
+    */
+    const std::size_t byte;
+
+  private:
+    parse_error(int id_, std::size_t byte_, const char* what_arg)
+        : exception(id_, what_arg), byte(byte_) {}
+
+    static std::string position_string(const position_t& pos)
+    {
+        return " at line " + std::to_string(pos.lines_read + 1) +
+               ", column " + std::to_string(pos.chars_read_current_line);
+    }
+};
+
+/*!
+@brief exception indicating errors with iterators
+
+This exception is thrown if iterators passed to a library function do not match
+the expected semantics.
+
+Exceptions have ids 2xx.
+
+name / id                           | example message | description
+----------------------------------- | --------------- | -------------------------
+json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
+json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from.
+json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid.
+json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid.
+json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range.
+json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key.
+json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
+json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
+json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered.
+json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
+
+@liveexample{The following code shows how an `invalid_iterator` exception can be
+caught.,invalid_iterator}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class invalid_iterator : public exception
+{
+  public:
+    static invalid_iterator create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("invalid_iterator", id_) + what_arg;
+        return invalid_iterator(id_, w.c_str());
+    }
+
+  private:
+    invalid_iterator(int id_, const char* what_arg)
+        : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating executing a member function with a wrong type
+
+This exception is thrown in case of a type error; that is, a library function is
+executed on a JSON value whose type does not match the expected semantics.
+
+Exceptions have ids 3xx.
+
+name / id                     | example message | description
+----------------------------- | --------------- | -------------------------
+json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead.
+json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types.
+json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&.
+json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types.
+json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types.
+json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types.
+json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types.
+json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types.
+json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
+json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
+json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
+json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types.
+json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
+json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
+json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
+json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. |
+json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) |
+
+@liveexample{The following code shows how a `type_error` exception can be
+caught.,type_error}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class type_error : public exception
+{
+  public:
+    static type_error create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("type_error", id_) + what_arg;
+        return type_error(id_, w.c_str());
+    }
+
+  private:
+    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating access out of the defined range
+
+This exception is thrown in case a library function is called on an input
+parameter that exceeds the expected range, for instance in case of array
+indices or nonexisting object keys.
+
+Exceptions have ids 4xx.
+
+name / id                       | example message | description
+------------------------------- | --------------- | -------------------------
+json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1.
+json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it.
+json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object.
+json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
+json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
+json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
+json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. |
+json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. |
+json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string |
+
+@liveexample{The following code shows how an `out_of_range` exception can be
+caught.,out_of_range}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class out_of_range : public exception
+{
+  public:
+    static out_of_range create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("out_of_range", id_) + what_arg;
+        return out_of_range(id_, w.c_str());
+    }
+
+  private:
+    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating other library errors
+
+This exception is thrown in case of errors that cannot be classified with the
+other exception types.
+
+Exceptions have ids 5xx.
+
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+
+@liveexample{The following code shows how an `other_error` exception can be
+caught.,other_error}
+
+@since version 3.0.0
+*/
+class other_error : public exception
+{
+  public:
+    static other_error create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("other_error", id_) + what_arg;
+        return other_error(id_, w.c_str());
+    }
+
+  private:
+    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+#include <array> // array
+#include <ciso646> // and
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////////////
+// JSON type enumeration //
+///////////////////////////
+
+/*!
+@brief the JSON type enumeration
+
+This enumeration collects the different JSON types. It is internally used to
+distinguish the stored values, and the functions @ref basic_json::is_null(),
+@ref basic_json::is_object(), @ref basic_json::is_array(),
+@ref basic_json::is_string(), @ref basic_json::is_boolean(),
+@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
+@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
+@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
+@ref basic_json::is_structured() rely on it.
+
+@note There are three enumeration entries (number_integer, number_unsigned, and
+number_float), because the library distinguishes these three types for numbers:
+@ref basic_json::number_unsigned_t is used for unsigned integers,
+@ref basic_json::number_integer_t is used for signed integers, and
+@ref basic_json::number_float_t is used for floating-point numbers or to
+approximate integers which do not fit in the limits of their respective type.
+
+@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON
+value with the default value for a given type
+
+@since version 1.0.0
+*/
+enum class value_t : std::uint8_t
+{
+    null,             ///< null value
+    object,           ///< object (unordered set of name/value pairs)
+    array,            ///< array (ordered collection of values)
+    string,           ///< string value
+    boolean,          ///< boolean value
+    number_integer,   ///< number value (signed integer)
+    number_unsigned,  ///< number value (unsigned integer)
+    number_float,     ///< number value (floating-point)
+    discarded         ///< discarded by the the parser callback function
+};
+
+/*!
+@brief comparison operator for JSON types
+
+Returns an ordering that is similar to Python:
+- order: null < boolean < number < object < array < string
+- furthermore, each type is not smaller than itself
+- discarded values are not comparable
+
+@since version 1.0.0
+*/
+inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+{
+    static constexpr std::array<std::uint8_t, 8> order = {{
+            0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
+            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
+        }
+    };
+
+    const auto l_index = static_cast<std::size_t>(lhs);
+    const auto r_index = static_cast<std::size_t>(rhs);
+    return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index];
+}
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
+
+
+#include <algorithm> // transform
+#include <array> // array
+#include <ciso646> // and, not
+#include <forward_list> // forward_list
+#include <iterator> // inserter, front_inserter, end
+#include <map> // map
+#include <string> // string
+#include <tuple> // tuple, make_tuple
+#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible
+#include <unordered_map> // unordered_map
+#include <utility> // pair, declval
+#include <valarray> // valarray
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename std::nullptr_t& n)
+{
+    if (JSON_UNLIKELY(not j.is_null()))
+    {
+        JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name())));
+    }
+    n = nullptr;
+}
+
+// overloads for basic_json template parameters
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t<std::is_arithmetic<ArithmeticType>::value and
+                     not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+                     int> = 0>
+void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+
+        default:
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
+    }
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
+{
+    if (JSON_UNLIKELY(not j.is_boolean()))
+    {
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name())));
+    }
+    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
+{
+    if (JSON_UNLIKELY(not j.is_string()))
+    {
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
+    }
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template <
+    typename BasicJsonType, typename ConstructibleStringType,
+    enable_if_t <
+        is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and
+        not std::is_same<typename BasicJsonType::string_t,
+                         ConstructibleStringType>::value,
+        int > = 0 >
+void from_json(const BasicJsonType& j, ConstructibleStringType& s)
+{
+    if (JSON_UNLIKELY(not j.is_string()))
+    {
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
+    }
+
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void from_json(const BasicJsonType& j, EnumType& e)
+{
+    typename std::underlying_type<EnumType>::type val;
+    get_arithmetic_value(j, val);
+    e = static_cast<EnumType>(val);
+}
+
+// forward_list doesn't have an insert method
+template<typename BasicJsonType, typename T, typename Allocator,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    std::transform(j.rbegin(), j.rend(),
+                   std::front_inserter(l), [](const BasicJsonType & i)
+    {
+        return i.template get<T>();
+    });
+}
+
+// valarray doesn't have an insert method
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::valarray<T>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    l.resize(j.size());
+    std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l));
+}
+
+template<typename BasicJsonType>
+void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/)
+{
+    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
+}
+
+template <typename BasicJsonType, typename T, std::size_t N>
+auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,
+                          priority_tag<2> /*unused*/)
+-> decltype(j.template get<T>(), void())
+{
+    for (std::size_t i = 0; i < N; ++i)
+    {
+        arr[i] = j.at(i).template get<T>();
+    }
+}
+
+template<typename BasicJsonType, typename ConstructibleArrayType>
+auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)
+-> decltype(
+    arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),
+    j.template get<typename ConstructibleArrayType::value_type>(),
+    void())
+{
+    using std::end;
+
+    arr.reserve(j.size());
+    std::transform(j.begin(), j.end(),
+                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename ConstructibleArrayType::value_type>();
+    });
+}
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,
+                          priority_tag<0> /*unused*/)
+{
+    using std::end;
+
+    std::transform(
+        j.begin(), j.end(), std::inserter(arr, end(arr)),
+        [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename ConstructibleArrayType::value_type>();
+    });
+}
+
+template <typename BasicJsonType, typename ConstructibleArrayType,
+          enable_if_t <
+              is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_basic_json<ConstructibleArrayType>::value,
+              int > = 0 >
+
+auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)
+-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),
+j.template get<typename ConstructibleArrayType::value_type>(),
+void())
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " +
+                                      std::string(j.type_name())));
+    }
+
+    from_json_array_impl(j, arr, priority_tag<3> {});
+}
+
+template<typename BasicJsonType, typename ConstructibleObjectType,
+         enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
+void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
+{
+    if (JSON_UNLIKELY(not j.is_object()))
+    {
+        JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name())));
+    }
+
+    auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+    using value_type = typename ConstructibleObjectType::value_type;
+    std::transform(
+        inner_object->begin(), inner_object->end(),
+        std::inserter(obj, obj.begin()),
+        [](typename BasicJsonType::object_t::value_type const & p)
+    {
+        return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
+    });
+}
+
+// overload for arithmetic types, not chosen for basic_json template arguments
+// (BooleanType, etc..); note: Is it really necessary to provide explicit
+// overloads for boolean_t etc. in case of a custom BooleanType which is not
+// an arithmetic type?
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t <
+             std::is_arithmetic<ArithmeticType>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+             int> = 0>
+void from_json(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+        case value_t::boolean:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
+            break;
+        }
+
+        default:
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
+    }
+}
+
+template<typename BasicJsonType, typename A1, typename A2>
+void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
+{
+    p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()};
+}
+
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/)
+{
+    t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
+}
+
+template<typename BasicJsonType, typename... Args>
+void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
+{
+    from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
+
+template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
+          typename = enable_if_t<not std::is_constructible<
+                                     typename BasicJsonType::string_t, Key>::value>>
+void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    for (const auto& p : j)
+    {
+        if (JSON_UNLIKELY(not p.is_array()))
+        {
+            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
+        }
+        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+    }
+}
+
+template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,
+          typename = enable_if_t<not std::is_constructible<
+                                     typename BasicJsonType::string_t, Key>::value>>
+void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    for (const auto& p : j)
+    {
+        if (JSON_UNLIKELY(not p.is_array()))
+        {
+            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
+        }
+        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+    }
+}
+
+struct from_json_fn
+{
+    template<typename BasicJsonType, typename T>
+    auto operator()(const BasicJsonType& j, T& val) const
+    noexcept(noexcept(from_json(j, val)))
+    -> decltype(from_json(j, val), void())
+    {
+        return from_json(j, val);
+    }
+};
+}  // namespace detail
+
+/// namespace to hold default `from_json` function
+/// to see why this is required:
+/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html
+namespace
+{
+constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
+} // namespace
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/conversions/to_json.hpp>
+
+
+#include <ciso646> // or, and, not
+#include <iterator> // begin, end
+#include <tuple> // tuple, get
+#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type
+#include <utility> // move, forward, declval, pair
+#include <valarray> // valarray
+#include <vector> // vector
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
+
+
+#include <cstddef> // size_t
+#include <string> // string, to_string
+#include <iterator> // input_iterator_tag
+#include <tuple> // tuple_size, get, tuple_element
+
+// #include <nlohmann/detail/value_t.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename IteratorType> class iteration_proxy_value
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+    using value_type = iteration_proxy_value;
+    using pointer = value_type * ;
+    using reference = value_type & ;
+    using iterator_category = std::input_iterator_tag;
+
+  private:
+    /// the iterator
+    IteratorType anchor;
+    /// an index for arrays (used to create key names)
+    std::size_t array_index = 0;
+    /// last stringified array index
+    mutable std::size_t array_index_last = 0;
+    /// a string representation of the array index
+    mutable std::string array_index_str = "0";
+    /// an empty string (to return a reference for primitive values)
+    const std::string empty_str = "";
+
+  public:
+    explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {}
+
+    /// dereference operator (needed for range-based for)
+    iteration_proxy_value& operator*()
+    {
+        return *this;
+    }
+
+    /// increment operator (needed for range-based for)
+    iteration_proxy_value& operator++()
+    {
+        ++anchor;
+        ++array_index;
+
+        return *this;
+    }
+
+    /// equality operator (needed for InputIterator)
+    bool operator==(const iteration_proxy_value& o) const noexcept
+    {
+        return anchor == o.anchor;
+    }
+
+    /// inequality operator (needed for range-based for)
+    bool operator!=(const iteration_proxy_value& o) const noexcept
+    {
+        return anchor != o.anchor;
+    }
+
+    /// return key of the iterator
+    const std::string& key() const
+    {
+        assert(anchor.m_object != nullptr);
+
+        switch (anchor.m_object->type())
+        {
+            // use integer array index as key
+            case value_t::array:
+            {
+                if (array_index != array_index_last)
+                {
+                    array_index_str = std::to_string(array_index);
+                    array_index_last = array_index;
+                }
+                return array_index_str;
+            }
+
+            // use key from the object
+            case value_t::object:
+                return anchor.key();
+
+            // use an empty key for all primitive types
+            default:
+                return empty_str;
+        }
+    }
+
+    /// return value of the iterator
+    typename IteratorType::reference value() const
+    {
+        return anchor.value();
+    }
+};
+
+/// proxy class for the items() function
+template<typename IteratorType> class iteration_proxy
+{
+  private:
+    /// the container to iterate
+    typename IteratorType::reference container;
+
+  public:
+    /// construct iteration proxy from a container
+    explicit iteration_proxy(typename IteratorType::reference cont) noexcept
+        : container(cont) {}
+
+    /// return iterator begin (needed for range-based for)
+    iteration_proxy_value<IteratorType> begin() noexcept
+    {
+        return iteration_proxy_value<IteratorType>(container.begin());
+    }
+
+    /// return iterator end (needed for range-based for)
+    iteration_proxy_value<IteratorType> end() noexcept
+    {
+        return iteration_proxy_value<IteratorType>(container.end());
+    }
+};
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key())
+{
+    return i.key();
+}
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value())
+{
+    return i.value();
+}
+}  // namespace detail
+}  // namespace nlohmann
+
+// The Addition to the STD Namespace is required to add
+// Structured Bindings Support to the iteration_proxy_value class
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+namespace std
+{
+template <typename IteratorType>
+class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>>
+            : public std::integral_constant<std::size_t, 2> {};
+
+template <std::size_t N, typename IteratorType>
+class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >>
+{
+  public:
+    using type = decltype(
+                     get<N>(std::declval <
+                            ::nlohmann::detail::iteration_proxy_value<IteratorType >> ()));
+};
+}
+
+namespace nlohmann
+{
+namespace detail
+{
+//////////////////
+// constructors //
+//////////////////
+
+template<value_t> struct external_constructor;
+
+template<>
+struct external_constructor<value_t::boolean>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
+    {
+        j.m_type = value_t::boolean;
+        j.m_value = b;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::string>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = s;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = std::move(s);
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleStringType,
+             enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleStringType& str)
+    {
+        j.m_type = value_t::string;
+        j.m_value.string = j.template create<typename BasicJsonType::string_t>(str);
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_float>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
+    {
+        j.m_type = value_t::number_float;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_unsigned>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
+    {
+        j.m_type = value_t::number_unsigned;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_integer>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
+    {
+        j.m_type = value_t::number_integer;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::array>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = arr;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = std::move(arr);
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleArrayType,
+             enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
+    {
+        using std::begin;
+        using std::end;
+        j.m_type = value_t::array;
+        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const std::vector<bool>& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->reserve(arr.size());
+        for (const bool x : arr)
+        {
+            j.m_value.array->push_back(x);
+        }
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename T,
+             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+    static void construct(BasicJsonType& j, const std::valarray<T>& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->resize(arr.size());
+        std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::object>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
+    {
+        j.m_type = value_t::object;
+        j.m_value = obj;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+    {
+        j.m_type = value_t::object;
+        j.m_value = std::move(obj);
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleObjectType,
+             enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
+    {
+        using std::begin;
+        using std::end;
+
+        j.m_type = value_t::object;
+        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
+        j.assert_invariant();
+    }
+};
+
+/////////////
+// to_json //
+/////////////
+
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
+void to_json(BasicJsonType& j, T b) noexcept
+{
+    external_constructor<value_t::boolean>::construct(j, b);
+}
+
+template<typename BasicJsonType, typename CompatibleString,
+         enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>
+void to_json(BasicJsonType& j, const CompatibleString& s)
+{
+    external_constructor<value_t::string>::construct(j, s);
+}
+
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+{
+    external_constructor<value_t::string>::construct(j, std::move(s));
+}
+
+template<typename BasicJsonType, typename FloatType,
+         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
+void to_json(BasicJsonType& j, FloatType val) noexcept
+{
+    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
+}
+
+template<typename BasicJsonType, typename CompatibleNumberUnsignedType,
+         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>
+void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
+{
+    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
+}
+
+template<typename BasicJsonType, typename CompatibleNumberIntegerType,
+         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>
+void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
+{
+    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
+}
+
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void to_json(BasicJsonType& j, EnumType e) noexcept
+{
+    using underlying_type = typename std::underlying_type<EnumType>::type;
+    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));
+}
+
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, const std::vector<bool>& e)
+{
+    external_constructor<value_t::array>::construct(j, e);
+}
+
+template <typename BasicJsonType, typename CompatibleArrayType,
+          enable_if_t<is_compatible_array_type<BasicJsonType,
+                      CompatibleArrayType>::value and
+                      not is_compatible_object_type<
+                          BasicJsonType, CompatibleArrayType>::value and
+                      not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and
+                      not is_basic_json<CompatibleArrayType>::value,
+                      int> = 0>
+void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
+
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+void to_json(BasicJsonType& j, const std::valarray<T>& arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template<typename BasicJsonType, typename CompatibleObjectType,
+         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0>
+void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
+{
+    external_constructor<value_t::object>::construct(j, obj);
+}
+
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+{
+    external_constructor<value_t::object>::construct(j, std::move(obj));
+}
+
+template <
+    typename BasicJsonType, typename T, std::size_t N,
+    enable_if_t<not std::is_constructible<typename BasicJsonType::string_t,
+                const T(&)[N]>::value,
+                int> = 0 >
+void to_json(BasicJsonType& j, const T(&arr)[N])
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
+
+template<typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::pair<Args...>& p)
+{
+    j = { p.first, p.second };
+}
+
+// for https://github.com/nlohmann/json/pull/1134
+template < typename BasicJsonType, typename T,
+           enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0>
+void to_json(BasicJsonType& j, const T& b)
+{
+    j = { {b.key(), b.value()} };
+}
+
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)
+{
+    j = { std::get<Idx>(t)... };
+}
+
+template<typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
+{
+    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
+
+struct to_json_fn
+{
+    template<typename BasicJsonType, typename T>
+    auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+    -> decltype(to_json(j, std::forward<T>(val)), void())
+    {
+        return to_json(j, std::forward<T>(val));
+    }
+};
+}  // namespace detail
+
+/// namespace to hold default `to_json` function
+namespace
+{
+constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
+} // namespace
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+
+#include <cassert> // assert
+#include <cstddef> // size_t
+#include <cstring> // strlen
+#include <istream> // istream
+#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
+#include <memory> // shared_ptr, make_shared, addressof
+#include <numeric> // accumulate
+#include <string> // string, char_traits
+#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
+#include <utility> // pair, declval
+#include <cstdio> //FILE *
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+/// the supported input formats
+enum class input_format_t { json, cbor, msgpack, ubjson, bson };
+
+////////////////////
+// input adapters //
+////////////////////
+
+/*!
+@brief abstract input adapter interface
+
+Produces a stream of std::char_traits<char>::int_type characters from a
+std::istream, a buffer, or some other input type. Accepts the return of
+exactly one non-EOF character for future input. The int_type characters
+returned consist of all valid char values as positive values (typically
+unsigned char), plus an EOF value outside that range, specified by the value
+of the function std::char_traits<char>::eof(). This value is typically -1, but
+could be any arbitrary value which is not a valid char value.
+*/
+struct input_adapter_protocol
+{
+    /// get a character [0,255] or std::char_traits<char>::eof().
+    virtual std::char_traits<char>::int_type get_character() = 0;
+    virtual ~input_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
+
+/*!
+Input adapter for stdio file access. This adapter read only 1 byte and do not use any
+ buffer. This adapter is a very low level adapter.
+*/
+class file_input_adapter : public input_adapter_protocol
+{
+  public:
+    explicit file_input_adapter(std::FILE* f)  noexcept
+        : m_file(f)
+    {}
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        return std::fgetc(m_file);
+    }
+  private:
+    /// the file pointer to read from
+    std::FILE* m_file;
+};
+
+
+/*!
+Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
+beginning of input. Does not support changing the underlying std::streambuf
+in mid-input. Maintains underlying std::istream and std::streambuf to support
+subsequent use of standard std::istream operations to process any input
+characters following those used in parsing the JSON input.  Clears the
+std::istream flags; any input errors (e.g., EOF) will be detected by the first
+subsequent call for input from the std::istream.
+*/
+class input_stream_adapter : public input_adapter_protocol
+{
+  public:
+    ~input_stream_adapter() override
+    {
+        // clear stream flags; we use underlying streambuf I/O, do not
+        // maintain ifstream flags, except eof
+        is.clear(is.rdstate() & std::ios::eofbit);
+    }
+
+    explicit input_stream_adapter(std::istream& i)
+        : is(i), sb(*i.rdbuf())
+    {}
+
+    // delete because of pointer members
+    input_stream_adapter(const input_stream_adapter&) = delete;
+    input_stream_adapter& operator=(input_stream_adapter&) = delete;
+    input_stream_adapter(input_stream_adapter&&) = delete;
+    input_stream_adapter& operator=(input_stream_adapter&&) = delete;
+
+    // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
+    // ensure that std::char_traits<char>::eof() and the character 0xFF do not
+    // end up as the same value, eg. 0xFFFFFFFF.
+    std::char_traits<char>::int_type get_character() override
+    {
+        auto res = sb.sbumpc();
+        // set eof manually, as we don't use the istream interface.
+        if (res == EOF)
+        {
+            is.clear(is.rdstate() | std::ios::eofbit);
+        }
+        return res;
+    }
+
+  private:
+    /// the associated input stream
+    std::istream& is;
+    std::streambuf& sb;
+};
+
+/// input adapter for buffer input
+class input_buffer_adapter : public input_adapter_protocol
+{
+  public:
+    input_buffer_adapter(const char* b, const std::size_t l) noexcept
+        : cursor(b), limit(b + l)
+    {}
+
+    // delete because of pointer members
+    input_buffer_adapter(const input_buffer_adapter&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+    input_buffer_adapter(input_buffer_adapter&&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&&) = delete;
+    ~input_buffer_adapter() override = default;
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        if (JSON_LIKELY(cursor < limit))
+        {
+            return std::char_traits<char>::to_int_type(*(cursor++));
+        }
+
+        return std::char_traits<char>::eof();
+    }
+
+  private:
+    /// pointer to the current character
+    const char* cursor;
+    /// pointer past the last character
+    const char* const limit;
+};
+
+template<typename WideStringType, size_t T>
+struct wide_string_input_helper
+{
+    // UTF-32
+    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
+    {
+        utf8_bytes_index = 0;
+
+        if (current_wchar == str.size())
+        {
+            utf8_bytes[0] = std::char_traits<char>::eof();
+            utf8_bytes_filled = 1;
+        }
+        else
+        {
+            // get the current character
+            const auto wc = static_cast<int>(str[current_wchar++]);
+
+            // UTF-32 to UTF-8 encoding
+            if (wc < 0x80)
+            {
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+            else if (wc <= 0x7FF)
+            {
+                utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F);
+                utf8_bytes[1] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 2;
+            }
+            else if (wc <= 0xFFFF)
+            {
+                utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F);
+                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[2] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 3;
+            }
+            else if (wc <= 0x10FFFF)
+            {
+                utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07);
+                utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F);
+                utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[3] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 4;
+            }
+            else
+            {
+                // unknown character
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+        }
+    }
+};
+
+template<typename WideStringType>
+struct wide_string_input_helper<WideStringType, 2>
+{
+    // UTF-16
+    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
+    {
+        utf8_bytes_index = 0;
+
+        if (current_wchar == str.size())
+        {
+            utf8_bytes[0] = std::char_traits<char>::eof();
+            utf8_bytes_filled = 1;
+        }
+        else
+        {
+            // get the current character
+            const auto wc = static_cast<int>(str[current_wchar++]);
+
+            // UTF-16 to UTF-8 encoding
+            if (wc < 0x80)
+            {
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+            else if (wc <= 0x7FF)
+            {
+                utf8_bytes[0] = 0xC0 | ((wc >> 6));
+                utf8_bytes[1] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 2;
+            }
+            else if (0xD800 > wc or wc >= 0xE000)
+            {
+                utf8_bytes[0] = 0xE0 | ((wc >> 12));
+                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[2] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 3;
+            }
+            else
+            {
+                if (current_wchar < str.size())
+                {
+                    const auto wc2 = static_cast<int>(str[current_wchar++]);
+                    const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF));
+                    utf8_bytes[0] = 0xf0 | (charcode >> 18);
+                    utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F);
+                    utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F);
+                    utf8_bytes[3] = 0x80 | (charcode & 0x3F);
+                    utf8_bytes_filled = 4;
+                }
+                else
+                {
+                    // unknown character
+                    ++current_wchar;
+                    utf8_bytes[0] = wc;
+                    utf8_bytes_filled = 1;
+                }
+            }
+        }
+    }
+};
+
+template<typename WideStringType>
+class wide_string_input_adapter : public input_adapter_protocol
+{
+  public:
+    explicit wide_string_input_adapter(const WideStringType& w)  noexcept
+        : str(w)
+    {}
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        // check if buffer needs to be filled
+        if (utf8_bytes_index == utf8_bytes_filled)
+        {
+            fill_buffer<sizeof(typename WideStringType::value_type)>();
+
+            assert(utf8_bytes_filled > 0);
+            assert(utf8_bytes_index == 0);
+        }
+
+        // use buffer
+        assert(utf8_bytes_filled > 0);
+        assert(utf8_bytes_index < utf8_bytes_filled);
+        return utf8_bytes[utf8_bytes_index++];
+    }
+
+  private:
+    template<size_t T>
+    void fill_buffer()
+    {
+        wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled);
+    }
+
+    /// the wstring to process
+    const WideStringType& str;
+
+    /// index of the current wchar in str
+    std::size_t current_wchar = 0;
+
+    /// a buffer for UTF-8 bytes
+    std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};
+
+    /// index to the utf8_codes array for the next valid byte
+    std::size_t utf8_bytes_index = 0;
+    /// number of valid bytes in the utf8_codes array
+    std::size_t utf8_bytes_filled = 0;
+};
+
+class input_adapter
+{
+  public:
+    // native support
+    input_adapter(std::FILE* file)
+        : ia(std::make_shared<file_input_adapter>(file)) {}
+    /// input adapter for input stream
+    input_adapter(std::istream& i)
+        : ia(std::make_shared<input_stream_adapter>(i)) {}
+
+    /// input adapter for input stream
+    input_adapter(std::istream&& i)
+        : ia(std::make_shared<input_stream_adapter>(i)) {}
+
+    input_adapter(const std::wstring& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {}
+
+    input_adapter(const std::u16string& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {}
+
+    input_adapter(const std::u32string& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {}
+
+    /// input adapter for buffer
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b, std::size_t l)
+        : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
+
+    // derived support
+
+    /// input adapter for string literal
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b)
+        : input_adapter(reinterpret_cast<const char*>(b),
+                        std::strlen(reinterpret_cast<const char*>(b))) {}
+
+    /// input adapter for iterator range with contiguous storage
+    template<class IteratorType,
+             typename std::enable_if<
+                 std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
+                 int>::type = 0>
+    input_adapter(IteratorType first, IteratorType last)
+    {
+#ifndef NDEBUG
+        // assertion to check that the iterator range is indeed contiguous,
+        // see http://stackoverflow.com/a/35008842/266378 for more discussion
+        const auto is_contiguous = std::accumulate(
+                                       first, last, std::pair<bool, int>(true, 0),
+                                       [&first](std::pair<bool, int> res, decltype(*first) val)
+        {
+            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
+            return res;
+        }).first;
+        assert(is_contiguous);
+#endif
+
+        // assertion to check that each element is 1 byte long
+        static_assert(
+            sizeof(typename iterator_traits<IteratorType>::value_type) == 1,
+            "each element in the iterator range must have the size of 1 byte");
+
+        const auto len = static_cast<size_t>(std::distance(first, last));
+        if (JSON_LIKELY(len > 0))
+        {
+            // there is at least one element: use the address of first
+            ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
+        }
+        else
+        {
+            // the address of first cannot be used: use nullptr
+            ia = std::make_shared<input_buffer_adapter>(nullptr, len);
+        }
+    }
+
+    /// input adapter for array
+    template<class T, std::size_t N>
+    input_adapter(T (&array)[N])
+        : input_adapter(std::begin(array), std::end(array)) {}
+
+    /// input adapter for contiguous container
+    template<class ContiguousContainer, typename
+             std::enable_if<not std::is_pointer<ContiguousContainer>::value and
+                            std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+                            int>::type = 0>
+    input_adapter(const ContiguousContainer& c)
+        : input_adapter(std::begin(c), std::end(c)) {}
+
+    operator input_adapter_t()
+    {
+        return ia;
+    }
+
+  private:
+    /// the actual adapter
+    input_adapter_t ia = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/lexer.hpp>
+
+
+#include <clocale> // localeconv
+#include <cstddef> // size_t
+#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
+#include <cstdio> // snprintf
+#include <initializer_list> // initializer_list
+#include <string> // char_traits, string
+#include <vector> // vector
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/position_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////
+// lexer //
+///////////
+
+/*!
+@brief lexical analysis
+
+This class organizes the lexical analysis during JSON deserialization.
+*/
+template<typename BasicJsonType>
+class lexer
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+  public:
+    /// token types for the parser
+    enum class token_type
+    {
+        uninitialized,    ///< indicating the scanner is uninitialized
+        literal_true,     ///< the `true` literal
+        literal_false,    ///< the `false` literal
+        literal_null,     ///< the `null` literal
+        value_string,     ///< a string -- use get_string() for actual value
+        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value
+        value_integer,    ///< a signed integer -- use get_number_integer() for actual value
+        value_float,      ///< an floating point number -- use get_number_float() for actual value
+        begin_array,      ///< the character for array begin `[`
+        begin_object,     ///< the character for object begin `{`
+        end_array,        ///< the character for array end `]`
+        end_object,       ///< the character for object end `}`
+        name_separator,   ///< the name separator `:`
+        value_separator,  ///< the value separator `,`
+        parse_error,      ///< indicating a parse error
+        end_of_input,     ///< indicating the end of the input buffer
+        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)
+    };
+
+    /// return name of values of type token_type (only used for errors)
+    static const char* token_type_name(const token_type t) noexcept
+    {
+        switch (t)
+        {
+            case token_type::uninitialized:
+                return "<uninitialized>";
+            case token_type::literal_true:
+                return "true literal";
+            case token_type::literal_false:
+                return "false literal";
+            case token_type::literal_null:
+                return "null literal";
+            case token_type::value_string:
+                return "string literal";
+            case lexer::token_type::value_unsigned:
+            case lexer::token_type::value_integer:
+            case lexer::token_type::value_float:
+                return "number literal";
+            case token_type::begin_array:
+                return "'['";
+            case token_type::begin_object:
+                return "'{'";
+            case token_type::end_array:
+                return "']'";
+            case token_type::end_object:
+                return "'}'";
+            case token_type::name_separator:
+                return "':'";
+            case token_type::value_separator:
+                return "','";
+            case token_type::parse_error:
+                return "<parse error>";
+            case token_type::end_of_input:
+                return "end of input";
+            case token_type::literal_or_value:
+                return "'[', '{', or a literal";
+            // LCOV_EXCL_START
+            default: // catch non-enum values
+                return "unknown token";
+                // LCOV_EXCL_STOP
+        }
+    }
+
+    explicit lexer(detail::input_adapter_t&& adapter)
+        : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
+
+    // delete because of pointer members
+    lexer(const lexer&) = delete;
+    lexer(lexer&&) = delete;
+    lexer& operator=(lexer&) = delete;
+    lexer& operator=(lexer&&) = delete;
+    ~lexer() = default;
+
+  private:
+    /////////////////////
+    // locales
+    /////////////////////
+
+    /// return the locale-dependent decimal point
+    static char get_decimal_point() noexcept
+    {
+        const auto loc = localeconv();
+        assert(loc != nullptr);
+        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
+    }
+
+    /////////////////////
+    // scan functions
+    /////////////////////
+
+    /*!
+    @brief get codepoint from 4 hex characters following `\u`
+
+    For input "\u c1 c2 c3 c4" the codepoint is:
+      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
+    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
+
+    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
+    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
+    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
+    between the ASCII value of the character and the desired integer value.
+
+    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
+            non-hex character)
+    */
+    int get_codepoint()
+    {
+        // this function only makes sense after reading `\u`
+        assert(current == 'u');
+        int codepoint = 0;
+
+        const auto factors = { 12, 8, 4, 0 };
+        for (const auto factor : factors)
+        {
+            get();
+
+            if (current >= '0' and current <= '9')
+            {
+                codepoint += ((current - 0x30) << factor);
+            }
+            else if (current >= 'A' and current <= 'F')
+            {
+                codepoint += ((current - 0x37) << factor);
+            }
+            else if (current >= 'a' and current <= 'f')
+            {
+                codepoint += ((current - 0x57) << factor);
+            }
+            else
+            {
+                return -1;
+            }
+        }
+
+        assert(0x0000 <= codepoint and codepoint <= 0xFFFF);
+        return codepoint;
+    }
+
+    /*!
+    @brief check if the next byte(s) are inside a given range
+
+    Adds the current byte and, for each passed range, reads a new byte and
+    checks if it is inside the range. If a violation was detected, set up an
+    error message and return false. Otherwise, return true.
+
+    @param[in] ranges  list of integers; interpreted as list of pairs of
+                       inclusive lower and upper bound, respectively
+
+    @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,
+         1, 2, or 3 pairs. This precondition is enforced by an assertion.
+
+    @return true if and only if no range violation was detected
+    */
+    bool next_byte_in_range(std::initializer_list<int> ranges)
+    {
+        assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6);
+        add(current);
+
+        for (auto range = ranges.begin(); range != ranges.end(); ++range)
+        {
+            get();
+            if (JSON_LIKELY(*range <= current and current <= *(++range)))
+            {
+                add(current);
+            }
+            else
+            {
+                error_message = "invalid string: ill-formed UTF-8 byte";
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief scan a string literal
+
+    This function scans a string according to Sect. 7 of RFC 7159. While
+    scanning, bytes are escaped and copied into buffer token_buffer. Then the
+    function returns successfully, token_buffer is *not* null-terminated (as it
+    may contain \0 bytes), and token_buffer.size() is the number of bytes in the
+    string.
+
+    @return token_type::value_string if string could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note In case of errors, variable error_message contains a textual
+          description.
+    */
+    token_type scan_string()
+    {
+        // reset token_buffer (ignore opening quote)
+        reset();
+
+        // we entered the function by reading an open quote
+        assert(current == '\"');
+
+        while (true)
+        {
+            // get next character
+            switch (get())
+            {
+                // end of file while parsing string
+                case std::char_traits<char>::eof():
+                {
+                    error_message = "invalid string: missing closing quote";
+                    return token_type::parse_error;
+                }
+
+                // closing quote
+                case '\"':
+                {
+                    return token_type::value_string;
+                }
+
+                // escapes
+                case '\\':
+                {
+                    switch (get())
+                    {
+                        // quotation mark
+                        case '\"':
+                            add('\"');
+                            break;
+                        // reverse solidus
+                        case '\\':
+                            add('\\');
+                            break;
+                        // solidus
+                        case '/':
+                            add('/');
+                            break;
+                        // backspace
+                        case 'b':
+                            add('\b');
+                            break;
+                        // form feed
+                        case 'f':
+                            add('\f');
+                            break;
+                        // line feed
+                        case 'n':
+                            add('\n');
+                            break;
+                        // carriage return
+                        case 'r':
+                            add('\r');
+                            break;
+                        // tab
+                        case 't':
+                            add('\t');
+                            break;
+
+                        // unicode escapes
+                        case 'u':
+                        {
+                            const int codepoint1 = get_codepoint();
+                            int codepoint = codepoint1; // start with codepoint1
+
+                            if (JSON_UNLIKELY(codepoint1 == -1))
+                            {
+                                error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                return token_type::parse_error;
+                            }
+
+                            // check if code point is a high surrogate
+                            if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
+                            {
+                                // expect next \uxxxx entry
+                                if (JSON_LIKELY(get() == '\\' and get() == 'u'))
+                                {
+                                    const int codepoint2 = get_codepoint();
+
+                                    if (JSON_UNLIKELY(codepoint2 == -1))
+                                    {
+                                        error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                        return token_type::parse_error;
+                                    }
+
+                                    // check if codepoint2 is a low surrogate
+                                    if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
+                                    {
+                                        // overwrite codepoint
+                                        codepoint =
+                                            // high surrogate occupies the most significant 22 bits
+                                            (codepoint1 << 10)
+                                            // low surrogate occupies the least significant 15 bits
+                                            + codepoint2
+                                            // there is still the 0xD800, 0xDC00 and 0x10000 noise
+                                            // in the result so we have to subtract with:
+                                            // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
+                                            - 0x35FDC00;
+                                    }
+                                    else
+                                    {
+                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                        return token_type::parse_error;
+                                    }
+                                }
+                                else
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+                            else
+                            {
+                                if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+
+                            // result of the above calculation yields a proper codepoint
+                            assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+                            // translate codepoint into bytes
+                            if (codepoint < 0x80)
+                            {
+                                // 1-byte characters: 0xxxxxxx (ASCII)
+                                add(codepoint);
+                            }
+                            else if (codepoint <= 0x7FF)
+                            {
+                                // 2-byte characters: 110xxxxx 10xxxxxx
+                                add(0xC0 | (codepoint >> 6));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+                            else if (codepoint <= 0xFFFF)
+                            {
+                                // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
+                                add(0xE0 | (codepoint >> 12));
+                                add(0x80 | ((codepoint >> 6) & 0x3F));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+                            else
+                            {
+                                // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                                add(0xF0 | (codepoint >> 18));
+                                add(0x80 | ((codepoint >> 12) & 0x3F));
+                                add(0x80 | ((codepoint >> 6) & 0x3F));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+
+                            break;
+                        }
+
+                        // other characters after escape
+                        default:
+                            error_message = "invalid string: forbidden character after backslash";
+                            return token_type::parse_error;
+                    }
+
+                    break;
+                }
+
+                // invalid control characters
+                case 0x00:
+                {
+                    error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000";
+                    return token_type::parse_error;
+                }
+
+                case 0x01:
+                {
+                    error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001";
+                    return token_type::parse_error;
+                }
+
+                case 0x02:
+                {
+                    error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002";
+                    return token_type::parse_error;
+                }
+
+                case 0x03:
+                {
+                    error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003";
+                    return token_type::parse_error;
+                }
+
+                case 0x04:
+                {
+                    error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004";
+                    return token_type::parse_error;
+                }
+
+                case 0x05:
+                {
+                    error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005";
+                    return token_type::parse_error;
+                }
+
+                case 0x06:
+                {
+                    error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006";
+                    return token_type::parse_error;
+                }
+
+                case 0x07:
+                {
+                    error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007";
+                    return token_type::parse_error;
+                }
+
+                case 0x08:
+                {
+                    error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b";
+                    return token_type::parse_error;
+                }
+
+                case 0x09:
+                {
+                    error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t";
+                    return token_type::parse_error;
+                }
+
+                case 0x0A:
+                {
+                    error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n";
+                    return token_type::parse_error;
+                }
+
+                case 0x0B:
+                {
+                    error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B";
+                    return token_type::parse_error;
+                }
+
+                case 0x0C:
+                {
+                    error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f";
+                    return token_type::parse_error;
+                }
+
+                case 0x0D:
+                {
+                    error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r";
+                    return token_type::parse_error;
+                }
+
+                case 0x0E:
+                {
+                    error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E";
+                    return token_type::parse_error;
+                }
+
+                case 0x0F:
+                {
+                    error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F";
+                    return token_type::parse_error;
+                }
+
+                case 0x10:
+                {
+                    error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010";
+                    return token_type::parse_error;
+                }
+
+                case 0x11:
+                {
+                    error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011";
+                    return token_type::parse_error;
+                }
+
+                case 0x12:
+                {
+                    error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012";
+                    return token_type::parse_error;
+                }
+
+                case 0x13:
+                {
+                    error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013";
+                    return token_type::parse_error;
+                }
+
+                case 0x14:
+                {
+                    error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014";
+                    return token_type::parse_error;
+                }
+
+                case 0x15:
+                {
+                    error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015";
+                    return token_type::parse_error;
+                }
+
+                case 0x16:
+                {
+                    error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016";
+                    return token_type::parse_error;
+                }
+
+                case 0x17:
+                {
+                    error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017";
+                    return token_type::parse_error;
+                }
+
+                case 0x18:
+                {
+                    error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018";
+                    return token_type::parse_error;
+                }
+
+                case 0x19:
+                {
+                    error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019";
+                    return token_type::parse_error;
+                }
+
+                case 0x1A:
+                {
+                    error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A";
+                    return token_type::parse_error;
+                }
+
+                case 0x1B:
+                {
+                    error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B";
+                    return token_type::parse_error;
+                }
+
+                case 0x1C:
+                {
+                    error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C";
+                    return token_type::parse_error;
+                }
+
+                case 0x1D:
+                {
+                    error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D";
+                    return token_type::parse_error;
+                }
+
+                case 0x1E:
+                {
+                    error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E";
+                    return token_type::parse_error;
+                }
+
+                case 0x1F:
+                {
+                    error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F";
+                    return token_type::parse_error;
+                }
+
+                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
+                case 0x20:
+                case 0x21:
+                case 0x23:
+                case 0x24:
+                case 0x25:
+                case 0x26:
+                case 0x27:
+                case 0x28:
+                case 0x29:
+                case 0x2A:
+                case 0x2B:
+                case 0x2C:
+                case 0x2D:
+                case 0x2E:
+                case 0x2F:
+                case 0x30:
+                case 0x31:
+                case 0x32:
+                case 0x33:
+                case 0x34:
+                case 0x35:
+                case 0x36:
+                case 0x37:
+                case 0x38:
+                case 0x39:
+                case 0x3A:
+                case 0x3B:
+                case 0x3C:
+                case 0x3D:
+                case 0x3E:
+                case 0x3F:
+                case 0x40:
+                case 0x41:
+                case 0x42:
+                case 0x43:
+                case 0x44:
+                case 0x45:
+                case 0x46:
+                case 0x47:
+                case 0x48:
+                case 0x49:
+                case 0x4A:
+                case 0x4B:
+                case 0x4C:
+                case 0x4D:
+                case 0x4E:
+                case 0x4F:
+                case 0x50:
+                case 0x51:
+                case 0x52:
+                case 0x53:
+                case 0x54:
+                case 0x55:
+                case 0x56:
+                case 0x57:
+                case 0x58:
+                case 0x59:
+                case 0x5A:
+                case 0x5B:
+                case 0x5D:
+                case 0x5E:
+                case 0x5F:
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6A:
+                case 0x6B:
+                case 0x6C:
+                case 0x6D:
+                case 0x6E:
+                case 0x6F:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                case 0x78:
+                case 0x79:
+                case 0x7A:
+                case 0x7B:
+                case 0x7C:
+                case 0x7D:
+                case 0x7E:
+                case 0x7F:
+                {
+                    add(current);
+                    break;
+                }
+
+                // U+0080..U+07FF: bytes C2..DF 80..BF
+                case 0xC2:
+                case 0xC3:
+                case 0xC4:
+                case 0xC5:
+                case 0xC6:
+                case 0xC7:
+                case 0xC8:
+                case 0xC9:
+                case 0xCA:
+                case 0xCB:
+                case 0xCC:
+                case 0xCD:
+                case 0xCE:
+                case 0xCF:
+                case 0xD0:
+                case 0xD1:
+                case 0xD2:
+                case 0xD3:
+                case 0xD4:
+                case 0xD5:
+                case 0xD6:
+                case 0xD7:
+                case 0xD8:
+                case 0xD9:
+                case 0xDA:
+                case 0xDB:
+                case 0xDC:
+                case 0xDD:
+                case 0xDE:
+                case 0xDF:
+                {
+                    if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
+                case 0xE0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
+                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
+                case 0xE1:
+                case 0xE2:
+                case 0xE3:
+                case 0xE4:
+                case 0xE5:
+                case 0xE6:
+                case 0xE7:
+                case 0xE8:
+                case 0xE9:
+                case 0xEA:
+                case 0xEB:
+                case 0xEC:
+                case 0xEE:
+                case 0xEF:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
+                case 0xED:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
+                case 0xF0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
+                case 0xF1:
+                case 0xF2:
+                case 0xF3:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
+                case 0xF4:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // remaining bytes (80..C1 and F5..FF) are ill-formed
+                default:
+                {
+                    error_message = "invalid string: ill-formed UTF-8 byte";
+                    return token_type::parse_error;
+                }
+            }
+        }
+    }
+
+    static void strtof(float& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtof(str, endptr);
+    }
+
+    static void strtof(double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtod(str, endptr);
+    }
+
+    static void strtof(long double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtold(str, endptr);
+    }
+
+    /*!
+    @brief scan a number literal
+
+    This function scans a string according to Sect. 6 of RFC 7159.
+
+    The function is realized with a deterministic finite state machine derived
+    from the grammar described in RFC 7159. Starting in state "init", the
+    input is read and used to determined the next state. Only state "done"
+    accepts the number. State "error" is a trap state to model errors. In the
+    table below, "anything" means any character but the ones listed before.
+
+    state    | 0        | 1-9      | e E      | +       | -       | .        | anything
+    ---------|----------|----------|----------|---------|---------|----------|-----------
+    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
+    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
+    zero     | done     | done     | exponent | done    | done    | decimal1 | done
+    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
+    decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
+    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
+    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
+    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
+    any2     | any2     | any2     | done     | done    | done    | done     | done
+
+    The state machine is realized with one label per state (prefixed with
+    "scan_number_") and `goto` statements between them. The state machine
+    contains cycles, but any cycle can be left when EOF is read. Therefore,
+    the function is guaranteed to terminate.
+
+    During scanning, the read bytes are stored in token_buffer. This string is
+    then converted to a signed integer, an unsigned integer, or a
+    floating-point number.
+
+    @return token_type::value_unsigned, token_type::value_integer, or
+            token_type::value_float if number could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note The scanner is independent of the current locale. Internally, the
+          locale's decimal point is used instead of `.` to work with the
+          locale-dependent converters.
+    */
+    token_type scan_number()  // lgtm [cpp/use-of-goto]
+    {
+        // reset token_buffer to store the number's bytes
+        reset();
+
+        // the type of the parsed number; initially set to unsigned; will be
+        // changed if minus sign, decimal point or exponent is read
+        token_type number_type = token_type::value_unsigned;
+
+        // state (init): we just found out we need to scan a number
+        switch (current)
+        {
+            case '-':
+            {
+                add(current);
+                goto scan_number_minus;
+            }
+
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            // LCOV_EXCL_START
+            default:
+            {
+                // all other characters are rejected outside scan_number()
+                assert(false);
+            }
+                // LCOV_EXCL_STOP
+        }
+
+scan_number_minus:
+        // state: we just parsed a leading minus sign
+        number_type = token_type::value_integer;
+        switch (get())
+        {
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '-'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_zero:
+        // state: we just parse a zero (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_any1:
+        // state: we just parsed a number 0-9 (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_decimal1:
+        // state: we just parsed a decimal point
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '.'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_decimal2:
+        // we just parsed at least one number after a decimal point
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_exponent:
+        // we just parsed an exponent
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '+':
+            case '-':
+            {
+                add(current);
+                goto scan_number_sign;
+            }
+
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message =
+                    "invalid number; expected '+', '-', or digit after exponent";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_sign:
+        // we just parsed an exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after exponent sign";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_any2:
+        // we just parsed a number after the exponent or exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_done:
+        // unget the character after the number (we only read it to know that
+        // we are done scanning a number)
+        unget();
+
+        char* endptr = nullptr;
+        errno = 0;
+
+        // try to parse integers first and fall back to floats
+        if (number_type == token_type::value_unsigned)
+        {
+            const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == token_buffer.data() + token_buffer.size());
+
+            if (errno == 0)
+            {
+                value_unsigned = static_cast<number_unsigned_t>(x);
+                if (value_unsigned == x)
+                {
+                    return token_type::value_unsigned;
+                }
+            }
+        }
+        else if (number_type == token_type::value_integer)
+        {
+            const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == token_buffer.data() + token_buffer.size());
+
+            if (errno == 0)
+            {
+                value_integer = static_cast<number_integer_t>(x);
+                if (value_integer == x)
+                {
+                    return token_type::value_integer;
+                }
+            }
+        }
+
+        // this code is reached if we parse a floating-point number or if an
+        // integer conversion above failed
+        strtof(value_float, token_buffer.data(), &endptr);
+
+        // we checked the number format before
+        assert(endptr == token_buffer.data() + token_buffer.size());
+
+        return token_type::value_float;
+    }
+
+    /*!
+    @param[in] literal_text  the literal text to expect
+    @param[in] length        the length of the passed literal text
+    @param[in] return_type   the token type to return on success
+    */
+    token_type scan_literal(const char* literal_text, const std::size_t length,
+                            token_type return_type)
+    {
+        assert(current == literal_text[0]);
+        for (std::size_t i = 1; i < length; ++i)
+        {
+            if (JSON_UNLIKELY(get() != literal_text[i]))
+            {
+                error_message = "invalid literal";
+                return token_type::parse_error;
+            }
+        }
+        return return_type;
+    }
+
+    /////////////////////
+    // input management
+    /////////////////////
+
+    /// reset token_buffer; current character is beginning of token
+    void reset() noexcept
+    {
+        token_buffer.clear();
+        token_string.clear();
+        token_string.push_back(std::char_traits<char>::to_char_type(current));
+    }
+
+    /*
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns a
+    `std::char_traits<char>::eof()` in that case.  Stores the scanned characters
+    for use in error messages.
+
+    @return character read from the input
+    */
+    std::char_traits<char>::int_type get()
+    {
+        ++position.chars_read_total;
+        ++position.chars_read_current_line;
+
+        if (next_unget)
+        {
+            // just reset the next_unget variable and work with current
+            next_unget = false;
+        }
+        else
+        {
+            current = ia->get_character();
+        }
+
+        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        {
+            token_string.push_back(std::char_traits<char>::to_char_type(current));
+        }
+
+        if (current == '\n')
+        {
+            ++position.lines_read;
+            ++position.chars_read_current_line = 0;
+        }
+
+        return current;
+    }
+
+    /*!
+    @brief unget current character (read it again on next get)
+
+    We implement unget by setting variable next_unget to true. The input is not
+    changed - we just simulate ungetting by modifying chars_read_total,
+    chars_read_current_line, and token_string. The next call to get() will
+    behave as if the unget character is read again.
+    */
+    void unget()
+    {
+        next_unget = true;
+
+        --position.chars_read_total;
+
+        // in case we "unget" a newline, we have to also decrement the lines_read
+        if (position.chars_read_current_line == 0)
+        {
+            if (position.lines_read > 0)
+            {
+                --position.lines_read;
+            }
+        }
+        else
+        {
+            --position.chars_read_current_line;
+        }
+
+        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        {
+            assert(token_string.size() != 0);
+            token_string.pop_back();
+        }
+    }
+
+    /// add a character to token_buffer
+    void add(int c)
+    {
+        token_buffer.push_back(std::char_traits<char>::to_char_type(c));
+    }
+
+  public:
+    /////////////////////
+    // value getters
+    /////////////////////
+
+    /// return integer value
+    constexpr number_integer_t get_number_integer() const noexcept
+    {
+        return value_integer;
+    }
+
+    /// return unsigned integer value
+    constexpr number_unsigned_t get_number_unsigned() const noexcept
+    {
+        return value_unsigned;
+    }
+
+    /// return floating-point value
+    constexpr number_float_t get_number_float() const noexcept
+    {
+        return value_float;
+    }
+
+    /// return current string value (implicitly resets the token; useful only once)
+    string_t& get_string()
+    {
+        return token_buffer;
+    }
+
+    /////////////////////
+    // diagnostics
+    /////////////////////
+
+    /// return position of last read token
+    constexpr position_t get_position() const noexcept
+    {
+        return position;
+    }
+
+    /// return the last read token (for errors only).  Will never contain EOF
+    /// (an arbitrary value that is not a valid char value, often -1), because
+    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
+    std::string get_token_string() const
+    {
+        // escape control characters
+        std::string result;
+        for (const auto c : token_string)
+        {
+            if ('\x00' <= c and c <= '\x1F')
+            {
+                // escape control characters
+                char cs[9];
+                (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c));
+                result += cs;
+            }
+            else
+            {
+                // add character as is
+                result.push_back(c);
+            }
+        }
+
+        return result;
+    }
+
+    /// return syntax error message
+    constexpr const char* get_error_message() const noexcept
+    {
+        return error_message;
+    }
+
+    /////////////////////
+    // actual scanner
+    /////////////////////
+
+    /*!
+    @brief skip the UTF-8 byte order mark
+    @return true iff there is no BOM or the correct BOM has been skipped
+    */
+    bool skip_bom()
+    {
+        if (get() == 0xEF)
+        {
+            // check if we completely parse the BOM
+            return get() == 0xBB and get() == 0xBF;
+        }
+
+        // the first character is not the beginning of the BOM; unget it to
+        // process is later
+        unget();
+        return true;
+    }
+
+    token_type scan()
+    {
+        // initially, skip the BOM
+        if (position.chars_read_total == 0 and not skip_bom())
+        {
+            error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
+            return token_type::parse_error;
+        }
+
+        // read next character and ignore whitespace
+        do
+        {
+            get();
+        }
+        while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
+
+        switch (current)
+        {
+            // structural characters
+            case '[':
+                return token_type::begin_array;
+            case ']':
+                return token_type::end_array;
+            case '{':
+                return token_type::begin_object;
+            case '}':
+                return token_type::end_object;
+            case ':':
+                return token_type::name_separator;
+            case ',':
+                return token_type::value_separator;
+
+            // literals
+            case 't':
+                return scan_literal("true", 4, token_type::literal_true);
+            case 'f':
+                return scan_literal("false", 5, token_type::literal_false);
+            case 'n':
+                return scan_literal("null", 4, token_type::literal_null);
+
+            // string
+            case '\"':
+                return scan_string();
+
+            // number
+            case '-':
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+                return scan_number();
+
+            // end of input (the null byte is needed when parsing from
+            // string literals)
+            case '\0':
+            case std::char_traits<char>::eof():
+                return token_type::end_of_input;
+
+            // error
+            default:
+                error_message = "invalid literal";
+                return token_type::parse_error;
+        }
+    }
+
+  private:
+    /// input adapter
+    detail::input_adapter_t ia = nullptr;
+
+    /// the current character
+    std::char_traits<char>::int_type current = std::char_traits<char>::eof();
+
+    /// whether the next get() call should just return current
+    bool next_unget = false;
+
+    /// the start position of the current token
+    position_t position;
+
+    /// raw input token string (for error messages)
+    std::vector<char> token_string {};
+
+    /// buffer for variable-length tokens (numbers, strings)
+    string_t token_buffer {};
+
+    /// a description of occurred lexer errors
+    const char* error_message = "";
+
+    // number values
+    number_integer_t value_integer = 0;
+    number_unsigned_t value_unsigned = 0;
+    number_float_t value_float = 0;
+
+    /// the decimal point
+    const char decimal_point_char = '.';
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/parser.hpp>
+
+
+#include <cassert> // assert
+#include <cmath> // isfinite
+#include <cstdint> // uint8_t
+#include <functional> // function
+#include <string> // string
+#include <utility> // move
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+
+#include <cstdint> // size_t
+#include <utility> // declval
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename T>
+using null_function_t = decltype(std::declval<T&>().null());
+
+template <typename T>
+using boolean_function_t =
+    decltype(std::declval<T&>().boolean(std::declval<bool>()));
+
+template <typename T, typename Integer>
+using number_integer_function_t =
+    decltype(std::declval<T&>().number_integer(std::declval<Integer>()));
+
+template <typename T, typename Unsigned>
+using number_unsigned_function_t =
+    decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>()));
+
+template <typename T, typename Float, typename String>
+using number_float_function_t = decltype(std::declval<T&>().number_float(
+                                    std::declval<Float>(), std::declval<const String&>()));
+
+template <typename T, typename String>
+using string_function_t =
+    decltype(std::declval<T&>().string(std::declval<String&>()));
+
+template <typename T>
+using start_object_function_t =
+    decltype(std::declval<T&>().start_object(std::declval<std::size_t>()));
+
+template <typename T, typename String>
+using key_function_t =
+    decltype(std::declval<T&>().key(std::declval<String&>()));
+
+template <typename T>
+using end_object_function_t = decltype(std::declval<T&>().end_object());
+
+template <typename T>
+using start_array_function_t =
+    decltype(std::declval<T&>().start_array(std::declval<std::size_t>()));
+
+template <typename T>
+using end_array_function_t = decltype(std::declval<T&>().end_array());
+
+template <typename T, typename Exception>
+using parse_error_function_t = decltype(std::declval<T&>().parse_error(
+        std::declval<std::size_t>(), std::declval<const std::string&>(),
+        std::declval<const Exception&>()));
+
+template <typename SAX, typename BasicJsonType>
+struct is_sax
+{
+  private:
+    static_assert(is_basic_json<BasicJsonType>::value,
+                  "BasicJsonType must be of type basic_json<...>");
+
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using exception_t = typename BasicJsonType::exception;
+
+  public:
+    static constexpr bool value =
+        is_detected_exact<bool, null_function_t, SAX>::value &&
+        is_detected_exact<bool, boolean_function_t, SAX>::value &&
+        is_detected_exact<bool, number_integer_function_t, SAX,
+        number_integer_t>::value &&
+        is_detected_exact<bool, number_unsigned_function_t, SAX,
+        number_unsigned_t>::value &&
+        is_detected_exact<bool, number_float_function_t, SAX, number_float_t,
+        string_t>::value &&
+        is_detected_exact<bool, string_function_t, SAX, string_t>::value &&
+        is_detected_exact<bool, start_object_function_t, SAX>::value &&
+        is_detected_exact<bool, key_function_t, SAX, string_t>::value &&
+        is_detected_exact<bool, end_object_function_t, SAX>::value &&
+        is_detected_exact<bool, start_array_function_t, SAX>::value &&
+        is_detected_exact<bool, end_array_function_t, SAX>::value &&
+        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;
+};
+
+template <typename SAX, typename BasicJsonType>
+struct is_sax_static_asserts
+{
+  private:
+    static_assert(is_basic_json<BasicJsonType>::value,
+                  "BasicJsonType must be of type basic_json<...>");
+
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using exception_t = typename BasicJsonType::exception;
+
+  public:
+    static_assert(is_detected_exact<bool, null_function_t, SAX>::value,
+                  "Missing/invalid function: bool null()");
+    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+                  "Missing/invalid function: bool boolean(bool)");
+    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+                  "Missing/invalid function: bool boolean(bool)");
+    static_assert(
+        is_detected_exact<bool, number_integer_function_t, SAX,
+        number_integer_t>::value,
+        "Missing/invalid function: bool number_integer(number_integer_t)");
+    static_assert(
+        is_detected_exact<bool, number_unsigned_function_t, SAX,
+        number_unsigned_t>::value,
+        "Missing/invalid function: bool number_unsigned(number_unsigned_t)");
+    static_assert(is_detected_exact<bool, number_float_function_t, SAX,
+                  number_float_t, string_t>::value,
+                  "Missing/invalid function: bool number_float(number_float_t, const string_t&)");
+    static_assert(
+        is_detected_exact<bool, string_function_t, SAX, string_t>::value,
+        "Missing/invalid function: bool string(string_t&)");
+    static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value,
+                  "Missing/invalid function: bool start_object(std::size_t)");
+    static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value,
+                  "Missing/invalid function: bool key(string_t&)");
+    static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value,
+                  "Missing/invalid function: bool end_object()");
+    static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value,
+                  "Missing/invalid function: bool start_array(std::size_t)");
+    static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value,
+                  "Missing/invalid function: bool end_array()");
+    static_assert(
+        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value,
+        "Missing/invalid function: bool parse_error(std::size_t, const "
+        "std::string&, const exception&)");
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+// #include <nlohmann/detail/input/parser.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+
+namespace nlohmann
+{
+
+/*!
+@brief SAX interface
+
+This class describes the SAX interface used by @ref nlohmann::json::sax_parse.
+Each function is called in different situations while the input is parsed. The
+boolean return value informs the parser whether to continue processing the
+input.
+*/
+template<typename BasicJsonType>
+struct json_sax
+{
+    /// type for (signed) integers
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    /// type for unsigned integers
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    /// type for floating-point numbers
+    using number_float_t = typename BasicJsonType::number_float_t;
+    /// type for strings
+    using string_t = typename BasicJsonType::string_t;
+
+    /*!
+    @brief a null value was read
+    @return whether parsing should proceed
+    */
+    virtual bool null() = 0;
+
+    /*!
+    @brief a boolean value was read
+    @param[in] val  boolean value
+    @return whether parsing should proceed
+    */
+    virtual bool boolean(bool val) = 0;
+
+    /*!
+    @brief an integer number was read
+    @param[in] val  integer value
+    @return whether parsing should proceed
+    */
+    virtual bool number_integer(number_integer_t val) = 0;
+
+    /*!
+    @brief an unsigned integer number was read
+    @param[in] val  unsigned integer value
+    @return whether parsing should proceed
+    */
+    virtual bool number_unsigned(number_unsigned_t val) = 0;
+
+    /*!
+    @brief an floating-point number was read
+    @param[in] val  floating-point value
+    @param[in] s    raw token value
+    @return whether parsing should proceed
+    */
+    virtual bool number_float(number_float_t val, const string_t& s) = 0;
+
+    /*!
+    @brief a string was read
+    @param[in] val  string value
+    @return whether parsing should proceed
+    @note It is safe to move the passed string.
+    */
+    virtual bool string(string_t& val) = 0;
+
+    /*!
+    @brief the beginning of an object was read
+    @param[in] elements  number of object elements or -1 if unknown
+    @return whether parsing should proceed
+    @note binary formats may report the number of elements
+    */
+    virtual bool start_object(std::size_t elements) = 0;
+
+    /*!
+    @brief an object key was read
+    @param[in] val  object key
+    @return whether parsing should proceed
+    @note It is safe to move the passed string.
+    */
+    virtual bool key(string_t& val) = 0;
+
+    /*!
+    @brief the end of an object was read
+    @return whether parsing should proceed
+    */
+    virtual bool end_object() = 0;
+
+    /*!
+    @brief the beginning of an array was read
+    @param[in] elements  number of array elements or -1 if unknown
+    @return whether parsing should proceed
+    @note binary formats may report the number of elements
+    */
+    virtual bool start_array(std::size_t elements) = 0;
+
+    /*!
+    @brief the end of an array was read
+    @return whether parsing should proceed
+    */
+    virtual bool end_array() = 0;
+
+    /*!
+    @brief a parse error occurred
+    @param[in] position    the position in the input where the error occurs
+    @param[in] last_token  the last read token
+    @param[in] ex          an exception object describing the error
+    @return whether parsing should proceed (must return false)
+    */
+    virtual bool parse_error(std::size_t position,
+                             const std::string& last_token,
+                             const detail::exception& ex) = 0;
+
+    virtual ~json_sax() = default;
+};
+
+
+namespace detail
+{
+/*!
+@brief SAX implementation to create a JSON value from SAX events
+
+This class implements the @ref json_sax interface and processes the SAX events
+to create a JSON value which makes it basically a DOM parser. The structure or
+hierarchy of the JSON value is managed by the stack `ref_stack` which contains
+a pointer to the respective array or object for each recursion depth.
+
+After successful parsing, the value that is passed by reference to the
+constructor contains the parsed value.
+
+@tparam BasicJsonType  the JSON type
+*/
+template<typename BasicJsonType>
+class json_sax_dom_parser
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+    /*!
+    @param[in, out] r  reference to a JSON value that is manipulated while
+                       parsing
+    @param[in] allow_exceptions_  whether parse errors yield exceptions
+    */
+    explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true)
+        : root(r), allow_exceptions(allow_exceptions_)
+    {}
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool start_object(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
+
+        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+        {
+            JSON_THROW(out_of_range::create(408,
+                                            "excessive object size: " + std::to_string(len)));
+        }
+
+        return true;
+    }
+
+    bool key(string_t& val)
+    {
+        // add null at given key and store the reference for later
+        object_element = &(ref_stack.back()->m_value.object->operator[](val));
+        return true;
+    }
+
+    bool end_object()
+    {
+        ref_stack.pop_back();
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(BasicJsonType::value_t::array));
+
+        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+        {
+            JSON_THROW(out_of_range::create(408,
+                                            "excessive array size: " + std::to_string(len)));
+        }
+
+        return true;
+    }
+
+    bool end_array()
+    {
+        ref_stack.pop_back();
+        return true;
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+                     const detail::exception& ex)
+    {
+        errored = true;
+        if (allow_exceptions)
+        {
+            // determine the proper exception type from the id
+            switch ((ex.id / 100) % 100)
+            {
+                case 1:
+                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
+                case 4:
+                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
+                // LCOV_EXCL_START
+                case 2:
+                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
+                case 3:
+                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
+                case 5:
+                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
+                default:
+                    assert(false);
+                    // LCOV_EXCL_STOP
+            }
+        }
+        return false;
+    }
+
+    constexpr bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+    */
+    template<typename Value>
+    BasicJsonType* handle_value(Value&& v)
+    {
+        if (ref_stack.empty())
+        {
+            root = BasicJsonType(std::forward<Value>(v));
+            return &root;
+        }
+
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v));
+            return &(ref_stack.back()->m_value.array->back());
+        }
+        else
+        {
+            assert(object_element);
+            *object_element = BasicJsonType(std::forward<Value>(v));
+            return object_element;
+        }
+    }
+
+    /// the parsed JSON value
+    BasicJsonType& root;
+    /// stack to model hierarchy of values
+    std::vector<BasicJsonType*> ref_stack;
+    /// helper to hold the reference for the next object element
+    BasicJsonType* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+};
+
+template<typename BasicJsonType>
+class json_sax_dom_callback_parser
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using parser_callback_t = typename BasicJsonType::parser_callback_t;
+    using parse_event_t = typename BasicJsonType::parse_event_t;
+
+    json_sax_dom_callback_parser(BasicJsonType& r,
+                                 const parser_callback_t cb,
+                                 const bool allow_exceptions_ = true)
+        : root(r), callback(cb), allow_exceptions(allow_exceptions_)
+    {
+        keep_stack.push_back(true);
+    }
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool start_object(std::size_t len)
+    {
+        // check callback for object start
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded);
+        keep_stack.push_back(keep);
+
+        auto val = handle_value(BasicJsonType::value_t::object, true);
+        ref_stack.push_back(val.second);
+
+        // check object limit
+        if (ref_stack.back())
+        {
+            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+            {
+                JSON_THROW(out_of_range::create(408,
+                                                "excessive object size: " + std::to_string(len)));
+            }
+        }
+
+        return true;
+    }
+
+    bool key(string_t& val)
+    {
+        BasicJsonType k = BasicJsonType(val);
+
+        // check callback for key
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k);
+        key_keep_stack.push_back(keep);
+
+        // add discarded value at given key and store the reference for later
+        if (keep and ref_stack.back())
+        {
+            object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded);
+        }
+
+        return true;
+    }
+
+    bool end_object()
+    {
+        if (ref_stack.back())
+        {
+            if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back()))
+            {
+                // discard object
+                *ref_stack.back() = discarded;
+            }
+        }
+
+        assert(not ref_stack.empty());
+        assert(not keep_stack.empty());
+        ref_stack.pop_back();
+        keep_stack.pop_back();
+
+        if (not ref_stack.empty() and ref_stack.back())
+        {
+            // remove discarded value
+            if (ref_stack.back()->is_object())
+            {
+                for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
+                {
+                    if (it->is_discarded())
+                    {
+                        ref_stack.back()->erase(it);
+                        break;
+                    }
+                }
+            }
+        }
+
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded);
+        keep_stack.push_back(keep);
+
+        auto val = handle_value(BasicJsonType::value_t::array, true);
+        ref_stack.push_back(val.second);
+
+        // check array limit
+        if (ref_stack.back())
+        {
+            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+            {
+                JSON_THROW(out_of_range::create(408,
+                                                "excessive array size: " + std::to_string(len)));
+            }
+        }
+
+        return true;
+    }
+
+    bool end_array()
+    {
+        bool keep = true;
+
+        if (ref_stack.back())
+        {
+            keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
+            if (not keep)
+            {
+                // discard array
+                *ref_stack.back() = discarded;
+            }
+        }
+
+        assert(not ref_stack.empty());
+        assert(not keep_stack.empty());
+        ref_stack.pop_back();
+        keep_stack.pop_back();
+
+        // remove discarded value
+        if (not keep and not ref_stack.empty())
+        {
+            if (ref_stack.back()->is_array())
+            {
+                ref_stack.back()->m_value.array->pop_back();
+            }
+        }
+
+        return true;
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+                     const detail::exception& ex)
+    {
+        errored = true;
+        if (allow_exceptions)
+        {
+            // determine the proper exception type from the id
+            switch ((ex.id / 100) % 100)
+            {
+                case 1:
+                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
+                case 4:
+                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
+                // LCOV_EXCL_START
+                case 2:
+                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
+                case 3:
+                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
+                case 5:
+                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
+                default:
+                    assert(false);
+                    // LCOV_EXCL_STOP
+            }
+        }
+        return false;
+    }
+
+    constexpr bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @param[in] v  value to add to the JSON value we build during parsing
+    @param[in] skip_callback  whether we should skip calling the callback
+               function; this is required after start_array() and
+               start_object() SAX events, because otherwise we would call the
+               callback function with an empty array or object, respectively.
+
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+
+    @return pair of boolean (whether value should be kept) and pointer (to the
+            passed value in the ref_stack hierarchy; nullptr if not kept)
+    */
+    template<typename Value>
+    std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false)
+    {
+        assert(not keep_stack.empty());
+
+        // do not handle this value if we know it would be added to a discarded
+        // container
+        if (not keep_stack.back())
+        {
+            return {false, nullptr};
+        }
+
+        // create value
+        auto value = BasicJsonType(std::forward<Value>(v));
+
+        // check callback
+        const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value);
+
+        // do not handle this value if we just learnt it shall be discarded
+        if (not keep)
+        {
+            return {false, nullptr};
+        }
+
+        if (ref_stack.empty())
+        {
+            root = std::move(value);
+            return {true, &root};
+        }
+
+        // skip this value if we already decided to skip the parent
+        // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360)
+        if (not ref_stack.back())
+        {
+            return {false, nullptr};
+        }
+
+        // we now only expect arrays and objects
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            ref_stack.back()->m_value.array->push_back(std::move(value));
+            return {true, &(ref_stack.back()->m_value.array->back())};
+        }
+        else
+        {
+            // check if we should store an element for the current key
+            assert(not key_keep_stack.empty());
+            const bool store_element = key_keep_stack.back();
+            key_keep_stack.pop_back();
+
+            if (not store_element)
+            {
+                return {false, nullptr};
+            }
+
+            assert(object_element);
+            *object_element = std::move(value);
+            return {true, object_element};
+        }
+    }
+
+    /// the parsed JSON value
+    BasicJsonType& root;
+    /// stack to model hierarchy of values
+    std::vector<BasicJsonType*> ref_stack;
+    /// stack to manage which values to keep
+    std::vector<bool> keep_stack;
+    /// stack to manage which object keys to keep
+    std::vector<bool> key_keep_stack;
+    /// helper to hold the reference for the next object element
+    BasicJsonType* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// callback function
+    const parser_callback_t callback = nullptr;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+    /// a discarded value for the callback
+    BasicJsonType discarded = BasicJsonType::value_t::discarded;
+};
+
+template<typename BasicJsonType>
+class json_sax_acceptor
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+    bool null()
+    {
+        return true;
+    }
+
+    bool boolean(bool /*unused*/)
+    {
+        return true;
+    }
+
+    bool number_integer(number_integer_t /*unused*/)
+    {
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t /*unused*/)
+    {
+        return true;
+    }
+
+    bool number_float(number_float_t /*unused*/, const string_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool string(string_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool start_object(std::size_t  /*unused*/ = std::size_t(-1))
+    {
+        return true;
+    }
+
+    bool key(string_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool end_object()
+    {
+        return true;
+    }
+
+    bool start_array(std::size_t  /*unused*/ = std::size_t(-1))
+    {
+        return true;
+    }
+
+    bool end_array()
+    {
+        return true;
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/)
+    {
+        return false;
+    }
+};
+}  // namespace detail
+
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/lexer.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+////////////
+// parser //
+////////////
+
+/*!
+@brief syntax analysis
+
+This class implements a recursive decent parser.
+*/
+template<typename BasicJsonType>
+class parser
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using lexer_t = lexer<BasicJsonType>;
+    using token_type = typename lexer_t::token_type;
+
+  public:
+    enum class parse_event_t : uint8_t
+    {
+        /// the parser read `{` and started to process a JSON object
+        object_start,
+        /// the parser read `}` and finished processing a JSON object
+        object_end,
+        /// the parser read `[` and started to process a JSON array
+        array_start,
+        /// the parser read `]` and finished processing a JSON array
+        array_end,
+        /// the parser read a key of a value in an object
+        key,
+        /// the parser finished reading a JSON value
+        value
+    };
+
+    using parser_callback_t =
+        std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>;
+
+    /// a parser reading from an input adapter
+    explicit parser(detail::input_adapter_t&& adapter,
+                    const parser_callback_t cb = nullptr,
+                    const bool allow_exceptions_ = true)
+        : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_)
+    {
+        // read first token
+        get_token();
+    }
+
+    /*!
+    @brief public parser interface
+
+    @param[in] strict      whether to expect the last token to be EOF
+    @param[in,out] result  parsed JSON value
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    */
+    void parse(const bool strict, BasicJsonType& result)
+    {
+        if (callback)
+        {
+            json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions);
+            sax_parse_internal(&sdp);
+            result.assert_invariant();
+
+            // in strict mode, input must be completely read
+            if (strict and (get_token() != token_type::end_of_input))
+            {
+                sdp.parse_error(m_lexer.get_position(),
+                                m_lexer.get_token_string(),
+                                parse_error::create(101, m_lexer.get_position(),
+                                                    exception_message(token_type::end_of_input, "value")));
+            }
+
+            // in case of an error, return discarded value
+            if (sdp.is_errored())
+            {
+                result = value_t::discarded;
+                return;
+            }
+
+            // set top-level value to null if it was discarded by the callback
+            // function
+            if (result.is_discarded())
+            {
+                result = nullptr;
+            }
+        }
+        else
+        {
+            json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions);
+            sax_parse_internal(&sdp);
+            result.assert_invariant();
+
+            // in strict mode, input must be completely read
+            if (strict and (get_token() != token_type::end_of_input))
+            {
+                sdp.parse_error(m_lexer.get_position(),
+                                m_lexer.get_token_string(),
+                                parse_error::create(101, m_lexer.get_position(),
+                                                    exception_message(token_type::end_of_input, "value")));
+            }
+
+            // in case of an error, return discarded value
+            if (sdp.is_errored())
+            {
+                result = value_t::discarded;
+                return;
+            }
+        }
+    }
+
+    /*!
+    @brief public accept interface
+
+    @param[in] strict  whether to expect the last token to be EOF
+    @return whether the input is a proper JSON text
+    */
+    bool accept(const bool strict = true)
+    {
+        json_sax_acceptor<BasicJsonType> sax_acceptor;
+        return sax_parse(&sax_acceptor, strict);
+    }
+
+    template <typename SAX>
+    bool sax_parse(SAX* sax, const bool strict = true)
+    {
+        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+        const bool result = sax_parse_internal(sax);
+
+        // strict mode: next byte must be EOF
+        if (result and strict and (get_token() != token_type::end_of_input))
+        {
+            return sax->parse_error(m_lexer.get_position(),
+                                    m_lexer.get_token_string(),
+                                    parse_error::create(101, m_lexer.get_position(),
+                                            exception_message(token_type::end_of_input, "value")));
+        }
+
+        return result;
+    }
+
+  private:
+    template <typename SAX>
+    bool sax_parse_internal(SAX* sax)
+    {
+        // stack to remember the hierarchy of structured values we are parsing
+        // true = array; false = object
+        std::vector<bool> states;
+        // value to avoid a goto (see comment where set to true)
+        bool skip_to_state_evaluation = false;
+
+        while (true)
+        {
+            if (not skip_to_state_evaluation)
+            {
+                // invariant: get_token() was called before each iteration
+                switch (last_token)
+                {
+                    case token_type::begin_object:
+                    {
+                        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+                        {
+                            return false;
+                        }
+
+                        // closing } -> we are done
+                        if (get_token() == token_type::end_object)
+                        {
+                            if (JSON_UNLIKELY(not sax->end_object()))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
+
+                        // parse key
+                        if (JSON_UNLIKELY(last_token != token_type::value_string))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::value_string, "object key")));
+                        }
+                        if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
+                        {
+                            return false;
+                        }
+
+                        // parse separator (:)
+                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::name_separator, "object separator")));
+                        }
+
+                        // remember we are now inside an object
+                        states.push_back(false);
+
+                        // parse values
+                        get_token();
+                        continue;
+                    }
+
+                    case token_type::begin_array:
+                    {
+                        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
+                        {
+                            return false;
+                        }
+
+                        // closing ] -> we are done
+                        if (get_token() == token_type::end_array)
+                        {
+                            if (JSON_UNLIKELY(not sax->end_array()))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
+
+                        // remember we are now inside an array
+                        states.push_back(true);
+
+                        // parse values (no need to call get_token)
+                        continue;
+                    }
+
+                    case token_type::value_float:
+                    {
+                        const auto res = m_lexer.get_number_float();
+
+                        if (JSON_UNLIKELY(not std::isfinite(res)))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'"));
+                        }
+                        else
+                        {
+                            if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string())))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
+                    }
+
+                    case token_type::literal_false:
+                    {
+                        if (JSON_UNLIKELY(not sax->boolean(false)))
+                        {
+                            return false;
+                        }
+                        break;
+                    }
+
+                    case token_type::literal_null:
+                    {
+                        if (JSON_UNLIKELY(not sax->null()))
+                        {
+                            return false;
+                        }
+                        break;
+                    }
+
+                    case token_type::literal_true:
+                    {
+                        if (JSON_UNLIKELY(not sax->boolean(true)))
+                        {
+                            return false;
+                        }
+                        break;
+                    }
+
+                    case token_type::value_integer:
+                    {
+                        if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer())))
+                        {
+                            return false;
+                        }
+                        break;
+                    }
+
+                    case token_type::value_string:
+                    {
+                        if (JSON_UNLIKELY(not sax->string(m_lexer.get_string())))
+                        {
+                            return false;
+                        }
+                        break;
+                    }
+
+                    case token_type::value_unsigned:
+                    {
+                        if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned())))
+                        {
+                            return false;
+                        }
+                        break;
+                    }
+
+                    case token_type::parse_error:
+                    {
+                        // using "uninitialized" to avoid "expected" message
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::uninitialized, "value")));
+                    }
+
+                    default: // the last token was unexpected
+                    {
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::literal_or_value, "value")));
+                    }
+                }
+            }
+            else
+            {
+                skip_to_state_evaluation = false;
+            }
+
+            // we reached this line after we successfully parsed a value
+            if (states.empty())
+            {
+                // empty stack: we reached the end of the hierarchy: done
+                return true;
+            }
+            else
+            {
+                if (states.back())  // array
+                {
+                    // comma -> next value
+                    if (get_token() == token_type::value_separator)
+                    {
+                        // parse a new value
+                        get_token();
+                        continue;
+                    }
+
+                    // closing ]
+                    if (JSON_LIKELY(last_token == token_type::end_array))
+                    {
+                        if (JSON_UNLIKELY(not sax->end_array()))
+                        {
+                            return false;
+                        }
+
+                        // We are done with this array. Before we can parse a
+                        // new value, we need to evaluate the new state first.
+                        // By setting skip_to_state_evaluation to false, we
+                        // are effectively jumping to the beginning of this if.
+                        assert(not states.empty());
+                        states.pop_back();
+                        skip_to_state_evaluation = true;
+                        continue;
+                    }
+                    else
+                    {
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::end_array, "array")));
+                    }
+                }
+                else  // object
+                {
+                    // comma -> next value
+                    if (get_token() == token_type::value_separator)
+                    {
+                        // parse key
+                        if (JSON_UNLIKELY(get_token() != token_type::value_string))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::value_string, "object key")));
+                        }
+                        else
+                        {
+                            if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
+                            {
+                                return false;
+                            }
+                        }
+
+                        // parse separator (:)
+                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::name_separator, "object separator")));
+                        }
+
+                        // parse values
+                        get_token();
+                        continue;
+                    }
+
+                    // closing }
+                    if (JSON_LIKELY(last_token == token_type::end_object))
+                    {
+                        if (JSON_UNLIKELY(not sax->end_object()))
+                        {
+                            return false;
+                        }
+
+                        // We are done with this object. Before we can parse a
+                        // new value, we need to evaluate the new state first.
+                        // By setting skip_to_state_evaluation to false, we
+                        // are effectively jumping to the beginning of this if.
+                        assert(not states.empty());
+                        states.pop_back();
+                        skip_to_state_evaluation = true;
+                        continue;
+                    }
+                    else
+                    {
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::end_object, "object")));
+                    }
+                }
+            }
+        }
+    }
+
+    /// get next token from lexer
+    token_type get_token()
+    {
+        return (last_token = m_lexer.scan());
+    }
+
+    std::string exception_message(const token_type expected, const std::string& context)
+    {
+        std::string error_msg = "syntax error ";
+
+        if (not context.empty())
+        {
+            error_msg += "while parsing " + context + " ";
+        }
+
+        error_msg += "- ";
+
+        if (last_token == token_type::parse_error)
+        {
+            error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" +
+                         m_lexer.get_token_string() + "'";
+        }
+        else
+        {
+            error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token));
+        }
+
+        if (expected != token_type::uninitialized)
+        {
+            error_msg += "; expected " + std::string(lexer_t::token_type_name(expected));
+        }
+
+        return error_msg;
+    }
+
+  private:
+    /// callback function
+    const parser_callback_t callback = nullptr;
+    /// the type of the last read token
+    token_type last_token = token_type::uninitialized;
+    /// the lexer
+    lexer_t m_lexer;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+
+#include <cstddef> // ptrdiff_t
+#include <limits>  // numeric_limits
+
+namespace nlohmann
+{
+namespace detail
+{
+/*
+@brief an iterator for primitive JSON types
+
+This class models an iterator for primitive JSON types (boolean, number,
+string). It's only purpose is to allow the iterator/const_iterator classes
+to "iterate" over primitive values. Internally, the iterator is modeled by
+a `difference_type` variable. Value begin_value (`0`) models the begin,
+end_value (`1`) models past the end.
+*/
+class primitive_iterator_t
+{
+  private:
+    using difference_type = std::ptrdiff_t;
+    static constexpr difference_type begin_value = 0;
+    static constexpr difference_type end_value = begin_value + 1;
+
+    /// iterator as signed integer type
+    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
+
+  public:
+    constexpr difference_type get_value() const noexcept
+    {
+        return m_it;
+    }
+
+    /// set iterator to a defined beginning
+    void set_begin() noexcept
+    {
+        m_it = begin_value;
+    }
+
+    /// set iterator to a defined past the end
+    void set_end() noexcept
+    {
+        m_it = end_value;
+    }
+
+    /// return whether the iterator can be dereferenced
+    constexpr bool is_begin() const noexcept
+    {
+        return m_it == begin_value;
+    }
+
+    /// return whether the iterator is at end
+    constexpr bool is_end() const noexcept
+    {
+        return m_it == end_value;
+    }
+
+    friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it == rhs.m_it;
+    }
+
+    friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it < rhs.m_it;
+    }
+
+    primitive_iterator_t operator+(difference_type n) noexcept
+    {
+        auto result = *this;
+        result += n;
+        return result;
+    }
+
+    friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it - rhs.m_it;
+    }
+
+    primitive_iterator_t& operator++() noexcept
+    {
+        ++m_it;
+        return *this;
+    }
+
+    primitive_iterator_t const operator++(int) noexcept
+    {
+        auto result = *this;
+        ++m_it;
+        return result;
+    }
+
+    primitive_iterator_t& operator--() noexcept
+    {
+        --m_it;
+        return *this;
+    }
+
+    primitive_iterator_t const operator--(int) noexcept
+    {
+        auto result = *this;
+        --m_it;
+        return result;
+    }
+
+    primitive_iterator_t& operator+=(difference_type n) noexcept
+    {
+        m_it += n;
+        return *this;
+    }
+
+    primitive_iterator_t& operator-=(difference_type n) noexcept
+    {
+        m_it -= n;
+        return *this;
+    }
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
+
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+/*!
+@brief an iterator value
+
+@note This structure could easily be a union, but MSVC currently does not allow
+unions members with complex constructors, see https://github.com/nlohmann/json/pull/105.
+*/
+template<typename BasicJsonType> struct internal_iterator
+{
+    /// iterator for JSON objects
+    typename BasicJsonType::object_t::iterator object_iterator {};
+    /// iterator for JSON arrays
+    typename BasicJsonType::array_t::iterator array_iterator {};
+    /// generic iterator for all other types
+    primitive_iterator_t primitive_iterator {};
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/iterators/iter_impl.hpp>
+
+
+#include <ciso646> // not
+#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next
+#include <type_traits> // conditional, is_const, remove_const
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+// forward declare, to be able to friend it later on
+template<typename IteratorType> class iteration_proxy;
+template<typename IteratorType> class iteration_proxy_value;
+
+/*!
+@brief a template for a bidirectional iterator for the @ref basic_json class
+This class implements a both iterators (iterator and const_iterator) for the
+@ref basic_json class.
+@note An iterator is called *initialized* when a pointer to a JSON value has
+      been set (e.g., by a constructor or a copy assignment). If the iterator is
+      default-constructed, it is *uninitialized* and most methods are undefined.
+      **The library uses assertions to detect calls on uninitialized iterators.**
+@requirement The class satisfies the following concept requirements:
+-
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
+  The iterator that can be moved can be moved in both directions (i.e.
+  incremented and decremented).
+@since version 1.0.0, simplified in version 2.0.9, change to bidirectional
+       iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
+*/
+template<typename BasicJsonType>
+class iter_impl
+{
+    /// allow basic_json to access private members
+    friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
+    friend BasicJsonType;
+    friend iteration_proxy<iter_impl>;
+    friend iteration_proxy_value<iter_impl>;
+
+    using object_t = typename BasicJsonType::object_t;
+    using array_t = typename BasicJsonType::array_t;
+    // make sure BasicJsonType is basic_json or const basic_json
+    static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,
+                  "iter_impl only accepts (const) basic_json");
+
+  public:
+
+    /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17.
+    /// The C++ Standard has never required user-defined iterators to derive from std::iterator.
+    /// A user-defined iterator should provide publicly accessible typedefs named
+    /// iterator_category, value_type, difference_type, pointer, and reference.
+    /// Note that value_type is required to be non-const, even for constant iterators.
+    using iterator_category = std::bidirectional_iterator_tag;
+
+    /// the type of the values when the iterator is dereferenced
+    using value_type = typename BasicJsonType::value_type;
+    /// a type to represent differences between iterators
+    using difference_type = typename BasicJsonType::difference_type;
+    /// defines a pointer to the type iterated over (value_type)
+    using pointer = typename std::conditional<std::is_const<BasicJsonType>::value,
+          typename BasicJsonType::const_pointer,
+          typename BasicJsonType::pointer>::type;
+    /// defines a reference to the type iterated over (value_type)
+    using reference =
+        typename std::conditional<std::is_const<BasicJsonType>::value,
+        typename BasicJsonType::const_reference,
+        typename BasicJsonType::reference>::type;
+
+    /// default constructor
+    iter_impl() = default;
+
+    /*!
+    @brief constructor for a given JSON instance
+    @param[in] object  pointer to a JSON object for this iterator
+    @pre object != nullptr
+    @post The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    explicit iter_impl(pointer object) noexcept : m_object(object)
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = typename object_t::iterator();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = typename array_t::iterator();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator = primitive_iterator_t();
+                break;
+            }
+        }
+    }
+
+    /*!
+    @note The conventional copy constructor and copy assignment are implicitly
+          defined. Combined with the following converting constructor and
+          assignment, they support: (1) copy from iterator to iterator, (2)
+          copy from const iterator to const iterator, and (3) conversion from
+          iterator to const iterator. However conversion from const iterator
+          to iterator is not defined.
+    */
+
+    /*!
+    @brief converting constructor
+    @param[in] other  non-const iterator to copy from
+    @note It is not checked whether @a other is initialized.
+    */
+    iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+        : m_object(other.m_object), m_it(other.m_it) {}
+
+    /*!
+    @brief converting assignment
+    @param[in,out] other  non-const iterator to copy from
+    @return const/non-const iterator
+    @note It is not checked whether @a other is initialized.
+    */
+    iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+    {
+        m_object = other.m_object;
+        m_it = other.m_it;
+        return *this;
+    }
+
+  private:
+    /*!
+    @brief set the iterator to the first value
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    void set_begin() noexcept
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = m_object->m_value.object->begin();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = m_object->m_value.array->begin();
+                break;
+            }
+
+            case value_t::null:
+            {
+                // set to end so begin()==end() is true: null is empty
+                m_it.primitive_iterator.set_end();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator.set_begin();
+                break;
+            }
+        }
+    }
+
+    /*!
+    @brief set the iterator past the last value
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    void set_end() noexcept
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = m_object->m_value.object->end();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = m_object->m_value.array->end();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator.set_end();
+                break;
+            }
+        }
+    }
+
+  public:
+    /*!
+    @brief return a reference to the value pointed to by the iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference operator*() const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                assert(m_it.object_iterator != m_object->m_value.object->end());
+                return m_it.object_iterator->second;
+            }
+
+            case value_t::array:
+            {
+                assert(m_it.array_iterator != m_object->m_value.array->end());
+                return *m_it.array_iterator;
+            }
+
+            case value_t::null:
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
+                {
+                    return *m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief dereference the iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    pointer operator->() const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                assert(m_it.object_iterator != m_object->m_value.object->end());
+                return &(m_it.object_iterator->second);
+            }
+
+            case value_t::array:
+            {
+                assert(m_it.array_iterator != m_object->m_value.array->end());
+                return &*m_it.array_iterator;
+            }
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
+                {
+                    return m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief post-increment (it++)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl const operator++(int)
+    {
+        auto result = *this;
+        ++(*this);
+        return result;
+    }
+
+    /*!
+    @brief pre-increment (++it)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator++()
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                std::advance(m_it.object_iterator, 1);
+                break;
+            }
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, 1);
+                break;
+            }
+
+            default:
+            {
+                ++m_it.primitive_iterator;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief post-decrement (it--)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl const operator--(int)
+    {
+        auto result = *this;
+        --(*this);
+        return result;
+    }
+
+    /*!
+    @brief pre-decrement (--it)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator--()
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                std::advance(m_it.object_iterator, -1);
+                break;
+            }
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, -1);
+                break;
+            }
+
+            default:
+            {
+                --m_it.primitive_iterator;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief  comparison: equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator==(const iter_impl& other) const
+    {
+        // if objects are not the same, the comparison is undefined
+        if (JSON_UNLIKELY(m_object != other.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
+        }
+
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                return (m_it.object_iterator == other.m_it.object_iterator);
+
+            case value_t::array:
+                return (m_it.array_iterator == other.m_it.array_iterator);
+
+            default:
+                return (m_it.primitive_iterator == other.m_it.primitive_iterator);
+        }
+    }
+
+    /*!
+    @brief  comparison: not equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator!=(const iter_impl& other) const
+    {
+        return not operator==(other);
+    }
+
+    /*!
+    @brief  comparison: smaller
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator<(const iter_impl& other) const
+    {
+        // if objects are not the same, the comparison is undefined
+        if (JSON_UNLIKELY(m_object != other.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
+        }
+
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators"));
+
+            case value_t::array:
+                return (m_it.array_iterator < other.m_it.array_iterator);
+
+            default:
+                return (m_it.primitive_iterator < other.m_it.primitive_iterator);
+        }
+    }
+
+    /*!
+    @brief  comparison: less than or equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator<=(const iter_impl& other) const
+    {
+        return not other.operator < (*this);
+    }
+
+    /*!
+    @brief  comparison: greater than
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator>(const iter_impl& other) const
+    {
+        return not operator<=(other);
+    }
+
+    /*!
+    @brief  comparison: greater than or equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator>=(const iter_impl& other) const
+    {
+        return not operator<(other);
+    }
+
+    /*!
+    @brief  add to iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator+=(difference_type i)
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, i);
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator += i;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief  subtract from iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator-=(difference_type i)
+    {
+        return operator+=(-i);
+    }
+
+    /*!
+    @brief  add to iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator+(difference_type i) const
+    {
+        auto result = *this;
+        result += i;
+        return result;
+    }
+
+    /*!
+    @brief  addition of distance and iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    friend iter_impl operator+(difference_type i, const iter_impl& it)
+    {
+        auto result = it;
+        result += i;
+        return result;
+    }
+
+    /*!
+    @brief  subtract from iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator-(difference_type i) const
+    {
+        auto result = *this;
+        result -= i;
+        return result;
+    }
+
+    /*!
+    @brief  return difference
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    difference_type operator-(const iter_impl& other) const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
+
+            case value_t::array:
+                return m_it.array_iterator - other.m_it.array_iterator;
+
+            default:
+                return m_it.primitive_iterator - other.m_it.primitive_iterator;
+        }
+    }
+
+    /*!
+    @brief  access to successor
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference operator[](difference_type n) const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators"));
+
+            case value_t::array:
+                return *std::next(m_it.array_iterator, n);
+
+            case value_t::null:
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.get_value() == -n))
+                {
+                    return *m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief  return the key of an object iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    const typename object_t::key_type& key() const
+    {
+        assert(m_object != nullptr);
+
+        if (JSON_LIKELY(m_object->is_object()))
+        {
+            return m_it.object_iterator->first;
+        }
+
+        JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators"));
+    }
+
+    /*!
+    @brief  return the value of an iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference value() const
+    {
+        return operator*();
+    }
+
+  private:
+    /// associated JSON instance
+    pointer m_object = nullptr;
+    /// the actual iterator of the associated instance
+    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it;
+};
+}  // namespace detail
+} // namespace nlohmann
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
+
+// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>
+
+
+#include <cstddef> // ptrdiff_t
+#include <iterator> // reverse_iterator
+#include <utility> // declval
+
+namespace nlohmann
+{
+namespace detail
+{
+//////////////////////
+// reverse_iterator //
+//////////////////////
+
+/*!
+@brief a template for a reverse iterator class
+
+@tparam Base the base iterator type to reverse. Valid types are @ref
+iterator (to create @ref reverse_iterator) and @ref const_iterator (to
+create @ref const_reverse_iterator).
+
+@requirement The class satisfies the following concept requirements:
+-
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
+  The iterator that can be moved can be moved in both directions (i.e.
+  incremented and decremented).
+- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator):
+  It is possible to write to the pointed-to element (only if @a Base is
+  @ref iterator).
+
+@since version 1.0.0
+*/
+template<typename Base>
+class json_reverse_iterator : public std::reverse_iterator<Base>
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+    /// shortcut to the reverse iterator adapter
+    using base_iterator = std::reverse_iterator<Base>;
+    /// the reference type for the pointed-to element
+    using reference = typename Base::reference;
+
+    /// create reverse iterator from iterator
+    explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
+        : base_iterator(it) {}
+
+    /// create reverse iterator from base class
+    explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
+
+    /// post-increment (it++)
+    json_reverse_iterator const operator++(int)
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
+    }
+
+    /// pre-increment (++it)
+    json_reverse_iterator& operator++()
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator++());
+    }
+
+    /// post-decrement (it--)
+    json_reverse_iterator const operator--(int)
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
+    }
+
+    /// pre-decrement (--it)
+    json_reverse_iterator& operator--()
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator--());
+    }
+
+    /// add to iterator
+    json_reverse_iterator& operator+=(difference_type i)
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
+    }
+
+    /// add to iterator
+    json_reverse_iterator operator+(difference_type i) const
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
+    }
+
+    /// subtract from iterator
+    json_reverse_iterator operator-(difference_type i) const
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
+    }
+
+    /// return difference
+    difference_type operator-(const json_reverse_iterator& other) const
+    {
+        return base_iterator(*this) - base_iterator(other);
+    }
+
+    /// access to successor
+    reference operator[](difference_type n) const
+    {
+        return *(this->operator+(n));
+    }
+
+    /// return the key of an object iterator
+    auto key() const -> decltype(std::declval<Base>().key())
+    {
+        auto it = --this->base();
+        return it.key();
+    }
+
+    /// return the value of an iterator
+    reference value() const
+    {
+        auto it = --this->base();
+        return it.operator * ();
+    }
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+
+#include <algorithm> // copy
+#include <cstddef> // size_t
+#include <ios> // streamsize
+#include <iterator> // back_inserter
+#include <memory> // shared_ptr, make_shared
+#include <ostream> // basic_ostream
+#include <string> // basic_string
+#include <vector> // vector
+
+namespace nlohmann
+{
+namespace detail
+{
+/// abstract output adapter interface
+template<typename CharType> struct output_adapter_protocol
+{
+    virtual void write_character(CharType c) = 0;
+    virtual void write_characters(const CharType* s, std::size_t length) = 0;
+    virtual ~output_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+template<typename CharType>
+using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
+
+/// output adapter for byte vectors
+template<typename CharType>
+class output_vector_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_vector_adapter(std::vector<CharType>& vec) noexcept
+        : v(vec)
+    {}
+
+    void write_character(CharType c) override
+    {
+        v.push_back(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        std::copy(s, s + length, std::back_inserter(v));
+    }
+
+  private:
+    std::vector<CharType>& v;
+};
+
+/// output adapter for output streams
+template<typename CharType>
+class output_stream_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept
+        : stream(s)
+    {}
+
+    void write_character(CharType c) override
+    {
+        stream.put(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        stream.write(s, static_cast<std::streamsize>(length));
+    }
+
+  private:
+    std::basic_ostream<CharType>& stream;
+};
+
+/// output adapter for basic_string
+template<typename CharType, typename StringType = std::basic_string<CharType>>
+class output_string_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_string_adapter(StringType& s) noexcept
+        : str(s)
+    {}
+
+    void write_character(CharType c) override
+    {
+        str.push_back(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        str.append(s, length);
+    }
+
+  private:
+    StringType& str;
+};
+
+template<typename CharType, typename StringType = std::basic_string<CharType>>
+class output_adapter
+{
+  public:
+    output_adapter(std::vector<CharType>& vec)
+        : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {}
+
+    output_adapter(std::basic_ostream<CharType>& s)
+        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
+
+    output_adapter(StringType& s)
+        : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {}
+
+    operator output_adapter_t<CharType>()
+    {
+        return oa;
+    }
+
+  private:
+    output_adapter_t<CharType> oa = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+
+
+#include <algorithm> // generate_n
+#include <array> // array
+#include <cassert> // assert
+#include <cmath> // ldexp
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstdio> // snprintf
+#include <cstring> // memcpy
+#include <iterator> // back_inserter
+#include <limits> // numeric_limits
+#include <string> // char_traits, string
+#include <utility> // make_pair, move
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// binary reader //
+///////////////////
+
+/*!
+@brief deserialization of CBOR, MessagePack, and UBJSON values
+*/
+template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>>
+class binary_reader
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using json_sax_t = SAX;
+
+  public:
+    /*!
+    @brief create a binary reader
+
+    @param[in] adapter  input adapter to read from
+    */
+    explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
+    {
+        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+        assert(ia);
+    }
+
+    /*!
+    @param[in] format  the binary format to parse
+    @param[in] sax_    a SAX event processor
+    @param[in] strict  whether to expect the input to be consumed completed
+
+    @return
+    */
+    bool sax_parse(const input_format_t format,
+                   json_sax_t* sax_,
+                   const bool strict = true)
+    {
+        sax = sax_;
+        bool result = false;
+
+        switch (format)
+        {
+            case input_format_t::bson:
+                result = parse_bson_internal();
+                break;
+
+            case input_format_t::cbor:
+                result = parse_cbor_internal();
+                break;
+
+            case input_format_t::msgpack:
+                result = parse_msgpack_internal();
+                break;
+
+            case input_format_t::ubjson:
+                result = parse_ubjson_internal();
+                break;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                // LCOV_EXCL_STOP
+        }
+
+        // strict mode: next byte must be EOF
+        if (result and strict)
+        {
+            if (format == input_format_t::ubjson)
+            {
+                get_ignore_noop();
+            }
+            else
+            {
+                get();
+            }
+
+            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            {
+                return sax->parse_error(chars_read, get_token_string(),
+                                        parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value")));
+            }
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief determine system byte order
+
+    @return true if and only if system's byte order is little endian
+
+    @note from http://stackoverflow.com/a/1001328/266378
+    */
+    static constexpr bool little_endianess(int num = 1) noexcept
+    {
+        return (*reinterpret_cast<char*>(&num) == 1);
+    }
+
+  private:
+    //////////
+    // BSON //
+    //////////
+
+    /*!
+    @brief Reads in a BSON-object and passes it to the SAX-parser.
+    @return whether a valid BSON-value was passed to the SAX parser
+    */
+    bool parse_bson_internal()
+    {
+        std::int32_t document_size;
+        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+        {
+            return false;
+        }
+
+        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false)))
+        {
+            return false;
+        }
+
+        return sax->end_object();
+    }
+
+    /*!
+    @brief Parses a C-style string from the BSON input.
+    @param[in, out] result  A reference to the string variable where the read
+                            string is to be stored.
+    @return `true` if the \x00-byte indicating the end of the string was
+             encountered before the EOF; false` indicates an unexpected EOF.
+    */
+    bool get_bson_cstr(string_t& result)
+    {
+        auto out = std::back_inserter(result);
+        while (true)
+        {
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring")))
+            {
+                return false;
+            }
+            if (current == 0x00)
+            {
+                return true;
+            }
+            *out++ = static_cast<char>(current);
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief Parses a zero-terminated string of length @a len from the BSON
+           input.
+    @param[in] len  The length (including the zero-byte at the end) of the
+                    string to be read.
+    @param[in, out] result  A reference to the string variable where the read
+                            string is to be stored.
+    @tparam NumberType The type of the length @a len
+    @pre len >= 1
+    @return `true` if the string was successfully parsed
+    */
+    template<typename NumberType>
+    bool get_bson_string(const NumberType len, string_t& result)
+    {
+        if (JSON_UNLIKELY(len < 1))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string")));
+        }
+
+        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof();
+    }
+
+    /*!
+    @brief Read a BSON document element of the given @a element_type.
+    @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html
+    @param[in] element_type_parse_position The position in the input stream,
+               where the `element_type` was read.
+    @warning Not all BSON element types are supported yet. An unsupported
+             @a element_type will give rise to a parse_error.114:
+             Unsupported BSON record type 0x...
+    @return whether a valid BSON-object/array was passed to the SAX parser
+    */
+    bool parse_bson_element_internal(const int element_type,
+                                     const std::size_t element_type_parse_position)
+    {
+        switch (element_type)
+        {
+            case 0x01: // double
+            {
+                double number;
+                return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0x02: // string
+            {
+                std::int32_t len;
+                string_t value;
+                return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value);
+            }
+
+            case 0x03: // object
+            {
+                return parse_bson_internal();
+            }
+
+            case 0x04: // array
+            {
+                return parse_bson_array();
+            }
+
+            case 0x08: // boolean
+            {
+                return sax->boolean(get() != 0);
+            }
+
+            case 0x0A: // null
+            {
+                return sax->null();
+            }
+
+            case 0x10: // int32
+            {
+                std::int32_t value;
+                return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value);
+            }
+
+            case 0x12: // int64
+            {
+                std::int64_t value;
+                return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value);
+            }
+
+            default: // anything else not supported (yet)
+            {
+                char cr[3];
+                (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type));
+                return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr)));
+            }
+        }
+    }
+
+    /*!
+    @brief Read a BSON element list (as specified in the BSON-spec)
+
+    The same binary layout is used for objects and arrays, hence it must be
+    indicated with the argument @a is_array which one is expected
+    (true --> array, false --> object).
+
+    @param[in] is_array Determines if the element list being read is to be
+                        treated as an object (@a is_array == false), or as an
+                        array (@a is_array == true).
+    @return whether a valid BSON-object/array was passed to the SAX parser
+    */
+    bool parse_bson_element_list(const bool is_array)
+    {
+        string_t key;
+        while (int element_type = get())
+        {
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list")))
+            {
+                return false;
+            }
+
+            const std::size_t element_type_parse_position = chars_read;
+            if (JSON_UNLIKELY(not get_bson_cstr(key)))
+            {
+                return false;
+            }
+
+            if (not is_array)
+            {
+                if (not sax->key(key))
+                {
+                    return false;
+                }
+            }
+
+            if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position)))
+            {
+                return false;
+            }
+
+            // get_bson_cstr only appends
+            key.clear();
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief Reads an array from the BSON input and passes it to the SAX-parser.
+    @return whether a valid BSON-array was passed to the SAX parser
+    */
+    bool parse_bson_array()
+    {
+        std::int32_t document_size;
+        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
+        {
+            return false;
+        }
+
+        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true)))
+        {
+            return false;
+        }
+
+        return sax->end_array();
+    }
+
+    //////////
+    // CBOR //
+    //////////
+
+    /*!
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+
+    @return whether a valid CBOR value was passed to the SAX parser
+    */
+    bool parse_cbor_internal(const bool get_char = true)
+    {
+        switch (get_char ? get() : current)
+        {
+            // EOF
+            case std::char_traits<char>::eof():
+                return unexpect_eof(input_format_t::cbor, "value");
+
+            // Integer 0x00..0x17 (0..23)
+            case 0x00:
+            case 0x01:
+            case 0x02:
+            case 0x03:
+            case 0x04:
+            case 0x05:
+            case 0x06:
+            case 0x07:
+            case 0x08:
+            case 0x09:
+            case 0x0A:
+            case 0x0B:
+            case 0x0C:
+            case 0x0D:
+            case 0x0E:
+            case 0x0F:
+            case 0x10:
+            case 0x11:
+            case 0x12:
+            case 0x13:
+            case 0x14:
+            case 0x15:
+            case 0x16:
+            case 0x17:
+                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
+
+            case 0x18: // Unsigned integer (one-byte uint8_t follows)
+            {
+                uint8_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
+
+            case 0x19: // Unsigned integer (two-byte uint16_t follows)
+            {
+                uint16_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
+
+            case 0x1A: // Unsigned integer (four-byte uint32_t follows)
+            {
+                uint32_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
+
+            case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
+            {
+                uint64_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
+
+            // Negative integer -1-0x00..-1-0x17 (-1..-24)
+            case 0x20:
+            case 0x21:
+            case 0x22:
+            case 0x23:
+            case 0x24:
+            case 0x25:
+            case 0x26:
+            case 0x27:
+            case 0x28:
+            case 0x29:
+            case 0x2A:
+            case 0x2B:
+            case 0x2C:
+            case 0x2D:
+            case 0x2E:
+            case 0x2F:
+            case 0x30:
+            case 0x31:
+            case 0x32:
+            case 0x33:
+            case 0x34:
+            case 0x35:
+            case 0x36:
+            case 0x37:
+                return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current));
+
+            case 0x38: // Negative integer (one-byte uint8_t follows)
+            {
+                uint8_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
+            }
+
+            case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
+            {
+                uint16_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
+            }
+
+            case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
+            {
+                uint32_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
+            }
+
+            case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
+            {
+                uint64_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1)
+                        - static_cast<number_integer_t>(number));
+            }
+
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+            case 0x7F: // UTF-8 string (indefinite length)
+            {
+                string_t s;
+                return get_cbor_string(s) and sax->string(s);
+            }
+
+            // array (0x00..0x17 data items follow)
+            case 0x80:
+            case 0x81:
+            case 0x82:
+            case 0x83:
+            case 0x84:
+            case 0x85:
+            case 0x86:
+            case 0x87:
+            case 0x88:
+            case 0x89:
+            case 0x8A:
+            case 0x8B:
+            case 0x8C:
+            case 0x8D:
+            case 0x8E:
+            case 0x8F:
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+                return get_cbor_array(static_cast<std::size_t>(current & 0x1F));
+
+            case 0x98: // array (one-byte uint8_t for n follows)
+            {
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x99: // array (two-byte uint16_t for n follow)
+            {
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x9A: // array (four-byte uint32_t for n follow)
+            {
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x9B: // array (eight-byte uint64_t for n follow)
+            {
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x9F: // array (indefinite length)
+                return get_cbor_array(std::size_t(-1));
+
+            // map (0x00..0x17 pairs of data items follow)
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+                return get_cbor_object(static_cast<std::size_t>(current & 0x1F));
+
+            case 0xB8: // map (one-byte uint8_t for n follows)
+            {
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xB9: // map (two-byte uint16_t for n follow)
+            {
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xBA: // map (four-byte uint32_t for n follow)
+            {
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xBB: // map (eight-byte uint64_t for n follow)
+            {
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xBF: // map (indefinite length)
+                return get_cbor_object(std::size_t(-1));
+
+            case 0xF4: // false
+                return sax->boolean(false);
+
+            case 0xF5: // true
+                return sax->boolean(true);
+
+            case 0xF6: // null
+                return sax->null();
+
+            case 0xF9: // Half-Precision Float (two-byte IEEE 754)
+            {
+                const int byte1_raw = get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
+                {
+                    return false;
+                }
+                const int byte2_raw = get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
+                {
+                    return false;
+                }
+
+                const auto byte1 = static_cast<unsigned char>(byte1_raw);
+                const auto byte2 = static_cast<unsigned char>(byte2_raw);
+
+                // code from RFC 7049, Appendix D, Figure 3:
+                // As half-precision floating-point numbers were only added
+                // to IEEE 754 in 2008, today's programming platforms often
+                // still only have limited support for them. It is very
+                // easy to include at least decoding support for them even
+                // without such support. An example of a small decoder for
+                // half-precision floating-point numbers in the C language
+                // is shown in Fig. 3.
+                const int half = (byte1 << 8) + byte2;
+                const double val = [&half]
+                {
+                    const int exp = (half >> 10) & 0x1F;
+                    const int mant = half & 0x3FF;
+                    assert(0 <= exp and exp <= 32);
+                    assert(0 <= mant and mant <= 1024);
+                    switch (exp)
+                    {
+                        case 0:
+                            return std::ldexp(mant, -24);
+                        case 31:
+                            return (mant == 0)
+                            ? std::numeric_limits<double>::infinity()
+                            : std::numeric_limits<double>::quiet_NaN();
+                        default:
+                            return std::ldexp(mant + 1024, exp - 25);
+                    }
+                }();
+                return sax->number_float((half & 0x8000) != 0
+                                         ? static_cast<number_float_t>(-val)
+                                         : static_cast<number_float_t>(val), "");
+            }
+
+            case 0xFA: // Single-Precision Float (four-byte IEEE 754)
+            {
+                float number;
+                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
+            {
+                double number;
+                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            default: // anything else (0xFF is handled inside the other types)
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value")));
+            }
+        }
+    }
+
+    /*!
+    @brief reads a CBOR string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+    Additionally, CBOR's strings with indefinite lengths are supported.
+
+    @param[out] result  created string
+
+    @return whether string creation completed
+    */
+    bool get_cbor_string(string_t& result)
+    {
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string")))
+        {
+            return false;
+        }
+
+        switch (current)
+        {
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            {
+                return get_string(input_format_t::cbor, current & 0x1F, result);
+            }
+
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            {
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            {
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+            {
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+            {
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x7F: // UTF-8 string (indefinite length)
+            {
+                while (get() != 0xFF)
+                {
+                    string_t chunk;
+                    if (not get_cbor_string(chunk))
+                    {
+                        return false;
+                    }
+                    result.append(chunk);
+                }
+                return true;
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string")));
+            }
+        }
+    }
+
+    /*!
+    @param[in] len  the length of the array or std::size_t(-1) for an
+                    array of indefinite size
+    @return whether array creation completed
+    */
+    bool get_cbor_array(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_array(len)))
+        {
+            return false;
+        }
+
+        if (len != std::size_t(-1))
+        {
+            for (std::size_t i = 0; i < len; ++i)
+            {
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+            }
+        }
+        else
+        {
+            while (get() != 0xFF)
+            {
+                if (JSON_UNLIKELY(not parse_cbor_internal(false)))
+                {
+                    return false;
+                }
+            }
+        }
+
+        return sax->end_array();
+    }
+
+    /*!
+    @param[in] len  the length of the object or std::size_t(-1) for an
+                    object of indefinite size
+    @return whether object creation completed
+    */
+    bool get_cbor_object(const std::size_t len)
+    {
+        if (not JSON_UNLIKELY(sax->start_object(len)))
+        {
+            return false;
+        }
+
+        string_t key;
+        if (len != std::size_t(-1))
+        {
+            for (std::size_t i = 0; i < len; ++i)
+            {
+                get();
+                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
+                {
+                    return false;
+                }
+
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+                key.clear();
+            }
+        }
+        else
+        {
+            while (get() != 0xFF)
+            {
+                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
+                {
+                    return false;
+                }
+
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+                key.clear();
+            }
+        }
+
+        return sax->end_object();
+    }
+
+    /////////////
+    // MsgPack //
+    /////////////
+
+    /*!
+    @return whether a valid MessagePack value was passed to the SAX parser
+    */
+    bool parse_msgpack_internal()
+    {
+        switch (get())
+        {
+            // EOF
+            case std::char_traits<char>::eof():
+                return unexpect_eof(input_format_t::msgpack, "value");
+
+            // positive fixint
+            case 0x00:
+            case 0x01:
+            case 0x02:
+            case 0x03:
+            case 0x04:
+            case 0x05:
+            case 0x06:
+            case 0x07:
+            case 0x08:
+            case 0x09:
+            case 0x0A:
+            case 0x0B:
+            case 0x0C:
+            case 0x0D:
+            case 0x0E:
+            case 0x0F:
+            case 0x10:
+            case 0x11:
+            case 0x12:
+            case 0x13:
+            case 0x14:
+            case 0x15:
+            case 0x16:
+            case 0x17:
+            case 0x18:
+            case 0x19:
+            case 0x1A:
+            case 0x1B:
+            case 0x1C:
+            case 0x1D:
+            case 0x1E:
+            case 0x1F:
+            case 0x20:
+            case 0x21:
+            case 0x22:
+            case 0x23:
+            case 0x24:
+            case 0x25:
+            case 0x26:
+            case 0x27:
+            case 0x28:
+            case 0x29:
+            case 0x2A:
+            case 0x2B:
+            case 0x2C:
+            case 0x2D:
+            case 0x2E:
+            case 0x2F:
+            case 0x30:
+            case 0x31:
+            case 0x32:
+            case 0x33:
+            case 0x34:
+            case 0x35:
+            case 0x36:
+            case 0x37:
+            case 0x38:
+            case 0x39:
+            case 0x3A:
+            case 0x3B:
+            case 0x3C:
+            case 0x3D:
+            case 0x3E:
+            case 0x3F:
+            case 0x40:
+            case 0x41:
+            case 0x42:
+            case 0x43:
+            case 0x44:
+            case 0x45:
+            case 0x46:
+            case 0x47:
+            case 0x48:
+            case 0x49:
+            case 0x4A:
+            case 0x4B:
+            case 0x4C:
+            case 0x4D:
+            case 0x4E:
+            case 0x4F:
+            case 0x50:
+            case 0x51:
+            case 0x52:
+            case 0x53:
+            case 0x54:
+            case 0x55:
+            case 0x56:
+            case 0x57:
+            case 0x58:
+            case 0x59:
+            case 0x5A:
+            case 0x5B:
+            case 0x5C:
+            case 0x5D:
+            case 0x5E:
+            case 0x5F:
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            case 0x78:
+            case 0x79:
+            case 0x7A:
+            case 0x7B:
+            case 0x7C:
+            case 0x7D:
+            case 0x7E:
+            case 0x7F:
+                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
+
+            // fixmap
+            case 0x80:
+            case 0x81:
+            case 0x82:
+            case 0x83:
+            case 0x84:
+            case 0x85:
+            case 0x86:
+            case 0x87:
+            case 0x88:
+            case 0x89:
+            case 0x8A:
+            case 0x8B:
+            case 0x8C:
+            case 0x8D:
+            case 0x8E:
+            case 0x8F:
+                return get_msgpack_object(static_cast<std::size_t>(current & 0x0F));
+
+            // fixarray
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+            case 0x98:
+            case 0x99:
+            case 0x9A:
+            case 0x9B:
+            case 0x9C:
+            case 0x9D:
+            case 0x9E:
+            case 0x9F:
+                return get_msgpack_array(static_cast<std::size_t>(current & 0x0F));
+
+            // fixstr
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            case 0xB8:
+            case 0xB9:
+            case 0xBA:
+            case 0xBB:
+            case 0xBC:
+            case 0xBD:
+            case 0xBE:
+            case 0xBF:
+            {
+                string_t s;
+                return get_msgpack_string(s) and sax->string(s);
+            }
+
+            case 0xC0: // nil
+                return sax->null();
+
+            case 0xC2: // false
+                return sax->boolean(false);
+
+            case 0xC3: // true
+                return sax->boolean(true);
+
+            case 0xCA: // float 32
+            {
+                float number;
+                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0xCB: // float 64
+            {
+                double number;
+                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0xCC: // uint 8
+            {
+                uint8_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xCD: // uint 16
+            {
+                uint16_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xCE: // uint 32
+            {
+                uint32_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xCF: // uint 64
+            {
+                uint64_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xD0: // int 8
+            {
+                int8_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD1: // int 16
+            {
+                int16_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD2: // int 32
+            {
+                int32_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD3: // int 64
+            {
+                int64_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD9: // str 8
+            case 0xDA: // str 16
+            case 0xDB: // str 32
+            {
+                string_t s;
+                return get_msgpack_string(s) and sax->string(s);
+            }
+
+            case 0xDC: // array 16
+            {
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
+            }
+
+            case 0xDD: // array 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
+            }
+
+            case 0xDE: // map 16
+            {
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xDF: // map 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
+            }
+
+            // negative fixint
+            case 0xE0:
+            case 0xE1:
+            case 0xE2:
+            case 0xE3:
+            case 0xE4:
+            case 0xE5:
+            case 0xE6:
+            case 0xE7:
+            case 0xE8:
+            case 0xE9:
+            case 0xEA:
+            case 0xEB:
+            case 0xEC:
+            case 0xED:
+            case 0xEE:
+            case 0xEF:
+            case 0xF0:
+            case 0xF1:
+            case 0xF2:
+            case 0xF3:
+            case 0xF4:
+            case 0xF5:
+            case 0xF6:
+            case 0xF7:
+            case 0xF8:
+            case 0xF9:
+            case 0xFA:
+            case 0xFB:
+            case 0xFC:
+            case 0xFD:
+            case 0xFE:
+            case 0xFF:
+                return sax->number_integer(static_cast<int8_t>(current));
+
+            default: // anything else
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value")));
+            }
+        }
+    }
+
+    /*!
+    @brief reads a MessagePack string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+
+    @param[out] result  created string
+
+    @return whether string creation completed
+    */
+    bool get_msgpack_string(string_t& result)
+    {
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string")))
+        {
+            return false;
+        }
+
+        switch (current)
+        {
+            // fixstr
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            case 0xB8:
+            case 0xB9:
+            case 0xBA:
+            case 0xBB:
+            case 0xBC:
+            case 0xBD:
+            case 0xBE:
+            case 0xBF:
+            {
+                return get_string(input_format_t::msgpack, current & 0x1F, result);
+            }
+
+            case 0xD9: // str 8
+            {
+                uint8_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
+
+            case 0xDA: // str 16
+            {
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
+
+            case 0xDB: // str 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string")));
+            }
+        }
+    }
+
+    /*!
+    @param[in] len  the length of the array
+    @return whether array creation completed
+    */
+    bool get_msgpack_array(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_array(len)))
+        {
+            return false;
+        }
+
+        for (std::size_t i = 0; i < len; ++i)
+        {
+            if (JSON_UNLIKELY(not parse_msgpack_internal()))
+            {
+                return false;
+            }
+        }
+
+        return sax->end_array();
+    }
+
+    /*!
+    @param[in] len  the length of the object
+    @return whether object creation completed
+    */
+    bool get_msgpack_object(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_object(len)))
+        {
+            return false;
+        }
+
+        string_t key;
+        for (std::size_t i = 0; i < len; ++i)
+        {
+            get();
+            if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key)))
+            {
+                return false;
+            }
+
+            if (JSON_UNLIKELY(not parse_msgpack_internal()))
+            {
+                return false;
+            }
+            key.clear();
+        }
+
+        return sax->end_object();
+    }
+
+    ////////////
+    // UBJSON //
+    ////////////
+
+    /*!
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+
+    @return whether a valid UBJSON value was passed to the SAX parser
+    */
+    bool parse_ubjson_internal(const bool get_char = true)
+    {
+        return get_ubjson_value(get_char ? get_ignore_noop() : current);
+    }
+
+    /*!
+    @brief reads a UBJSON string
+
+    This function is either called after reading the 'S' byte explicitly
+    indicating a string, or in case of an object key where the 'S' byte can be
+    left out.
+
+    @param[out] result   created string
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+
+    @return whether string creation completed
+    */
+    bool get_ubjson_string(string_t& result, const bool get_char = true)
+    {
+        if (get_char)
+        {
+            get();  // TODO: may we ignore N here?
+        }
+
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
+        {
+            return false;
+        }
+
+        switch (current)
+        {
+            case 'U':
+            {
+                uint8_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'i':
+            {
+                int8_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'I':
+            {
+                int16_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'l':
+            {
+                int32_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'L':
+            {
+                int64_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            default:
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string")));
+        }
+    }
+
+    /*!
+    @param[out] result  determined size
+    @return whether size determination completed
+    */
+    bool get_ubjson_size_value(std::size_t& result)
+    {
+        switch (get_ignore_noop())
+        {
+            case 'U':
+            {
+                uint8_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'i':
+            {
+                int8_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'I':
+            {
+                int16_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'l':
+            {
+                int32_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'L':
+            {
+                int64_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size")));
+            }
+        }
+    }
+
+    /*!
+    @brief determine the type and size for a container
+
+    In the optimized UBJSON format, a type and a size can be provided to allow
+    for a more compact representation.
+
+    @param[out] result  pair of the size and the type
+
+    @return whether pair creation completed
+    */
+    bool get_ubjson_size_type(std::pair<std::size_t, int>& result)
+    {
+        result.first = string_t::npos; // size
+        result.second = 0; // type
+
+        get_ignore_noop();
+
+        if (current == '$')
+        {
+            result.second = get();  // must not ignore 'N', because 'N' maybe the type
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type")))
+            {
+                return false;
+            }
+
+            get_ignore_noop();
+            if (JSON_UNLIKELY(current != '#'))
+            {
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
+                {
+                    return false;
+                }
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size")));
+            }
+
+            return get_ubjson_size_value(result.first);
+        }
+        else if (current == '#')
+        {
+            return get_ubjson_size_value(result.first);
+        }
+        return true;
+    }
+
+    /*!
+    @param prefix  the previously read or set type prefix
+    @return whether value creation completed
+    */
+    bool get_ubjson_value(const int prefix)
+    {
+        switch (prefix)
+        {
+            case std::char_traits<char>::eof():  // EOF
+                return unexpect_eof(input_format_t::ubjson, "value");
+
+            case 'T':  // true
+                return sax->boolean(true);
+            case 'F':  // false
+                return sax->boolean(false);
+
+            case 'Z':  // null
+                return sax->null();
+
+            case 'U':
+            {
+                uint8_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number);
+            }
+
+            case 'i':
+            {
+                int8_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'I':
+            {
+                int16_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'l':
+            {
+                int32_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'L':
+            {
+                int64_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'd':
+            {
+                float number;
+                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 'D':
+            {
+                double number;
+                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 'C':  // char
+            {
+                get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char")))
+                {
+                    return false;
+                }
+                if (JSON_UNLIKELY(current > 127))
+                {
+                    auto last_token = get_token_string();
+                    return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char")));
+                }
+                string_t s(1, static_cast<char>(current));
+                return sax->string(s);
+            }
+
+            case 'S':  // string
+            {
+                string_t s;
+                return get_ubjson_string(s) and sax->string(s);
+            }
+
+            case '[':  // array
+                return get_ubjson_array();
+
+            case '{':  // object
+                return get_ubjson_object();
+
+            default: // anything else
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value")));
+            }
+        }
+    }
+
+    /*!
+    @return whether array creation completed
+    */
+    bool get_ubjson_array()
+    {
+        std::pair<std::size_t, int> size_and_type;
+        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
+        {
+            return false;
+        }
+
+        if (size_and_type.first != string_t::npos)
+        {
+            if (JSON_UNLIKELY(not sax->start_array(size_and_type.first)))
+            {
+                return false;
+            }
+
+            if (size_and_type.second != 0)
+            {
+                if (size_and_type.second != 'N')
+                {
+                    for (std::size_t i = 0; i < size_and_type.first; ++i)
+                    {
+                        if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
+                        {
+                            return false;
+                        }
+                    }
+                }
+            }
+            else
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                    {
+                        return false;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
+            {
+                return false;
+            }
+
+            while (current != ']')
+            {
+                if (JSON_UNLIKELY(not parse_ubjson_internal(false)))
+                {
+                    return false;
+                }
+                get_ignore_noop();
+            }
+        }
+
+        return sax->end_array();
+    }
+
+    /*!
+    @return whether object creation completed
+    */
+    bool get_ubjson_object()
+    {
+        std::pair<std::size_t, int> size_and_type;
+        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
+        {
+            return false;
+        }
+
+        string_t key;
+        if (size_and_type.first != string_t::npos)
+        {
+            if (JSON_UNLIKELY(not sax->start_object(size_and_type.first)))
+            {
+                return false;
+            }
+
+            if (size_and_type.second != 0)
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
+                    {
+                        return false;
+                    }
+                    if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
+                    {
+                        return false;
+                    }
+                    key.clear();
+                }
+            }
+            else
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
+                    {
+                        return false;
+                    }
+                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                    {
+                        return false;
+                    }
+                    key.clear();
+                }
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+            {
+                return false;
+            }
+
+            while (current != '}')
+            {
+                if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key)))
+                {
+                    return false;
+                }
+                if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                {
+                    return false;
+                }
+                get_ignore_noop();
+                key.clear();
+            }
+        }
+
+        return sax->end_object();
+    }
+
+    ///////////////////////
+    // Utility functions //
+    ///////////////////////
+
+    /*!
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns a -'ve valued
+    `std::char_traits<char>::eof()` in that case.
+
+    @return character read from the input
+    */
+    int get()
+    {
+        ++chars_read;
+        return (current = ia->get_character());
+    }
+
+    /*!
+    @return character read from the input after ignoring all 'N' entries
+    */
+    int get_ignore_noop()
+    {
+        do
+        {
+            get();
+        }
+        while (current == 'N');
+
+        return current;
+    }
+
+    /*
+    @brief read a number from the input
+
+    @tparam NumberType the type of the number
+    @param[in] format   the current format (for diagnostics)
+    @param[out] result  number of type @a NumberType
+
+    @return whether conversion completed
+
+    @note This function needs to respect the system's endianess, because
+          bytes in CBOR, MessagePack, and UBJSON are stored in network order
+          (big endian) and therefore need reordering on little endian systems.
+    */
+    template<typename NumberType, bool InputIsLittleEndian = false>
+    bool get_number(const input_format_t format, NumberType& result)
+    {
+        // step 1: read input into array with system's byte order
+        std::array<uint8_t, sizeof(NumberType)> vec;
+        for (std::size_t i = 0; i < sizeof(NumberType); ++i)
+        {
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(format, "number")))
+            {
+                return false;
+            }
+
+            // reverse byte order prior to conversion if necessary
+            if (is_little_endian && !InputIsLittleEndian)
+            {
+                vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
+            }
+            else
+            {
+                vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE
+            }
+        }
+
+        // step 2: convert array into number of type T and return
+        std::memcpy(&result, vec.data(), sizeof(NumberType));
+        return true;
+    }
+
+    /*!
+    @brief create a string by reading characters from the input
+
+    @tparam NumberType the type of the number
+    @param[in] format the current format (for diagnostics)
+    @param[in] len number of characters to read
+    @param[out] result string created by reading @a len bytes
+
+    @return whether string creation completed
+
+    @note We can not reserve @a len bytes for the result, because @a len
+          may be too large. Usually, @ref unexpect_eof() detects the end of
+          the input before we run out of string memory.
+    */
+    template<typename NumberType>
+    bool get_string(const input_format_t format,
+                    const NumberType len,
+                    string_t& result)
+    {
+        bool success = true;
+        std::generate_n(std::back_inserter(result), len, [this, &success, &format]()
+        {
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(format, "string")))
+            {
+                success = false;
+            }
+            return static_cast<char>(current);
+        });
+        return success;
+    }
+
+    /*!
+    @param[in] format   the current format (for diagnostics)
+    @param[in] context  further context information (for diagnostics)
+    @return whether the last read character is not EOF
+    */
+    bool unexpect_eof(const input_format_t format, const char* context) const
+    {
+        if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+        {
+            return sax->parse_error(chars_read, "<end of file>",
+                                    parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context)));
+        }
+        return true;
+    }
+
+    /*!
+    @return a string representation of the last read byte
+    */
+    std::string get_token_string() const
+    {
+        char cr[3];
+        (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current));
+        return std::string{cr};
+    }
+
+    /*!
+    @param[in] format   the current format
+    @param[in] detail   a detailed error message
+    @param[in] context  further contect information
+    @return a message string to use in the parse_error exceptions
+    */
+    std::string exception_message(const input_format_t format,
+                                  const std::string& detail,
+                                  const std::string& context) const
+    {
+        std::string error_msg = "syntax error while parsing ";
+
+        switch (format)
+        {
+            case input_format_t::cbor:
+                error_msg += "CBOR";
+                break;
+
+            case input_format_t::msgpack:
+                error_msg += "MessagePack";
+                break;
+
+            case input_format_t::ubjson:
+                error_msg += "UBJSON";
+                break;
+
+            case input_format_t::bson:
+                error_msg += "BSON";
+                break;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                // LCOV_EXCL_STOP
+        }
+
+        return error_msg + " " + context + ": " + detail;
+    }
+
+  private:
+    /// input adapter
+    input_adapter_t ia = nullptr;
+
+    /// the current character
+    int current = std::char_traits<char>::eof();
+
+    /// the number of characters read
+    std::size_t chars_read = 0;
+
+    /// whether we can assume little endianess
+    const bool is_little_endian = little_endianess();
+
+    /// the SAX parser
+    json_sax_t* sax = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+
+
+#include <algorithm> // reverse
+#include <array> // array
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstring> // memcpy
+#include <limits> // numeric_limits
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// binary writer //
+///////////////////
+
+/*!
+@brief serialization to CBOR and MessagePack values
+*/
+template<typename BasicJsonType, typename CharType>
+class binary_writer
+{
+    using string_t = typename BasicJsonType::string_t;
+
+  public:
+    /*!
+    @brief create a binary writer
+
+    @param[in] adapter  output adapter to write to
+    */
+    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter)
+    {
+        assert(oa);
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    void write_bson(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::object:
+            {
+                write_bson_object(*j.m_value.object);
+                break;
+            }
+
+            default:
+            {
+                JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name())));
+            }
+        }
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    */
+    void write_cbor(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+            {
+                oa->write_character(to_char_type(0xF6));
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                oa->write_character(j.m_value.boolean
+                                    ? to_char_type(0xF5)
+                                    : to_char_type(0xF4));
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // CBOR does not differentiate between positive signed
+                    // integers and unsigned integers. Therefore, we used the
+                    // code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_integer <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x18));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x19));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x1A));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else
+                    {
+                        oa->write_character(to_char_type(0x1B));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    // The conversions below encode the sign in the first
+                    // byte, and the value is converted to a positive number.
+                    const auto positive_number = -1 - j.m_value.number_integer;
+                    if (j.m_value.number_integer >= -24)
+                    {
+                        write_number(static_cast<uint8_t>(0x20 + positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x38));
+                        write_number(static_cast<uint8_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x39));
+                        write_number(static_cast<uint16_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x3A));
+                        write_number(static_cast<uint32_t>(positive_number));
+                    }
+                    else
+                    {
+                        oa->write_character(to_char_type(0x3B));
+                        write_number(static_cast<uint64_t>(positive_number));
+                    }
+                }
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x18));
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x19));
+                    write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x1A));
+                    write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
+                }
+                else
+                {
+                    oa->write_character(to_char_type(0x1B));
+                    write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
+                }
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                oa->write_character(get_cbor_float_prefix(j.m_value.number_float));
+                write_number(j.m_value.number_float);
+                break;
+            }
+
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x60 + N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x78));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x79));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x7A));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x7B));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x80 + N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x98));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x99));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x9A));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x9B));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_cbor(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0xA0 + N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xB8));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xB9));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xBA));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xBB));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_cbor(el.first);
+                    write_cbor(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    */
+    void write_msgpack(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null: // nil
+            {
+                oa->write_character(to_char_type(0xC0));
+                break;
+            }
+
+            case value_t::boolean: // true and false
+            {
+                oa->write_character(j.m_value.boolean
+                                    ? to_char_type(0xC3)
+                                    : to_char_type(0xC2));
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // MessagePack does not differentiate between positive
+                    // signed integers and unsigned integers. Therefore, we used
+                    // the code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_unsigned < 128)
+                    {
+                        // positive fixnum
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        // uint 8
+                        oa->write_character(to_char_type(0xCC));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        // uint 16
+                        oa->write_character(to_char_type(0xCD));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        // uint 32
+                        oa->write_character(to_char_type(0xCE));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                    {
+                        // uint 64
+                        oa->write_character(to_char_type(0xCF));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    if (j.m_value.number_integer >= -32)
+                    {
+                        // negative fixnum
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                    {
+                        // int 8
+                        oa->write_character(to_char_type(0xD0));
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                    {
+                        // int 16
+                        oa->write_character(to_char_type(0xD1));
+                        write_number(static_cast<int16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                    {
+                        // int 32
+                        oa->write_character(to_char_type(0xD2));
+                        write_number(static_cast<int32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
+                    {
+                        // int 64
+                        oa->write_character(to_char_type(0xD3));
+                        write_number(static_cast<int64_t>(j.m_value.number_integer));
+                    }
+                }
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned < 128)
+                {
+                    // positive fixnum
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    // uint 8
+                    oa->write_character(to_char_type(0xCC));
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // uint 16
+                    oa->write_character(to_char_type(0xCD));
+                    write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // uint 32
+                    oa->write_character(to_char_type(0xCE));
+                    write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    // uint 64
+                    oa->write_character(to_char_type(0xCF));
+                    write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                }
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                oa->write_character(get_msgpack_float_prefix(j.m_value.number_float));
+                write_number(j.m_value.number_float);
+                break;
+            }
+
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 31)
+                {
+                    // fixstr
+                    write_number(static_cast<uint8_t>(0xA0 | N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    // str 8
+                    oa->write_character(to_char_type(0xD9));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // str 16
+                    oa->write_character(to_char_type(0xDA));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // str 32
+                    oa->write_character(to_char_type(0xDB));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 15)
+                {
+                    // fixarray
+                    write_number(static_cast<uint8_t>(0x90 | N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // array 16
+                    oa->write_character(to_char_type(0xDC));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // array 32
+                    oa->write_character(to_char_type(0xDD));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_msgpack(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 15)
+                {
+                    // fixmap
+                    write_number(static_cast<uint8_t>(0x80 | (N & 0xF)));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // map 16
+                    oa->write_character(to_char_type(0xDE));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // map 32
+                    oa->write_character(to_char_type(0xDF));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_msgpack(el.first);
+                    write_msgpack(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    @param[in] use_count   whether to use '#' prefixes (optimized format)
+    @param[in] use_type    whether to use '$' prefixes (optimized format)
+    @param[in] add_prefix  whether prefixes need to be used for this value
+    */
+    void write_ubjson(const BasicJsonType& j, const bool use_count,
+                      const bool use_type, const bool add_prefix = true)
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('Z'));
+                }
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(j.m_value.boolean
+                                        ? to_char_type('T')
+                                        : to_char_type('F'));
+                }
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix);
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix);
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix);
+                break;
+            }
+
+            case value_t::string:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('S'));
+                }
+                write_number_with_ubjson_prefix(j.m_value.string->size(), true);
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('['));
+                }
+
+                bool prefix_required = true;
+                if (use_type and not j.m_value.array->empty())
+                {
+                    assert(use_count);
+                    const CharType first_prefix = ubjson_prefix(j.front());
+                    const bool same_prefix = std::all_of(j.begin() + 1, j.end(),
+                                                         [this, first_prefix](const BasicJsonType & v)
+                    {
+                        return ubjson_prefix(v) == first_prefix;
+                    });
+
+                    if (same_prefix)
+                    {
+                        prefix_required = false;
+                        oa->write_character(to_char_type('$'));
+                        oa->write_character(first_prefix);
+                    }
+                }
+
+                if (use_count)
+                {
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.array->size(), true);
+                }
+
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_ubjson(el, use_count, use_type, prefix_required);
+                }
+
+                if (not use_count)
+                {
+                    oa->write_character(to_char_type(']'));
+                }
+
+                break;
+            }
+
+            case value_t::object:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('{'));
+                }
+
+                bool prefix_required = true;
+                if (use_type and not j.m_value.object->empty())
+                {
+                    assert(use_count);
+                    const CharType first_prefix = ubjson_prefix(j.front());
+                    const bool same_prefix = std::all_of(j.begin(), j.end(),
+                                                         [this, first_prefix](const BasicJsonType & v)
+                    {
+                        return ubjson_prefix(v) == first_prefix;
+                    });
+
+                    if (same_prefix)
+                    {
+                        prefix_required = false;
+                        oa->write_character(to_char_type('$'));
+                        oa->write_character(first_prefix);
+                    }
+                }
+
+                if (use_count)
+                {
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.object->size(), true);
+                }
+
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_number_with_ubjson_prefix(el.first.size(), true);
+                    oa->write_characters(
+                        reinterpret_cast<const CharType*>(el.first.c_str()),
+                        el.first.size());
+                    write_ubjson(el.second, use_count, use_type, prefix_required);
+                }
+
+                if (not use_count)
+                {
+                    oa->write_character(to_char_type('}'));
+                }
+
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+  private:
+    //////////
+    // BSON //
+    //////////
+
+    /*!
+    @return The size of a BSON document entry header, including the id marker
+            and the entry name size (and its null-terminator).
+    */
+    static std::size_t calc_bson_entry_header_size(const string_t& name)
+    {
+        const auto it = name.find(static_cast<typename string_t::value_type>(0));
+        if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos))
+        {
+            JSON_THROW(out_of_range::create(409,
+                                            "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")"));
+        }
+
+        return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
+    }
+
+    /*!
+    @brief Writes the given @a element_type and @a name to the output adapter
+    */
+    void write_bson_entry_header(const string_t& name,
+                                 const std::uint8_t element_type)
+    {
+        oa->write_character(to_char_type(element_type)); // boolean
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(name.c_str()),
+            name.size() + 1u);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and boolean value @a value
+    */
+    void write_bson_boolean(const string_t& name,
+                            const bool value)
+    {
+        write_bson_entry_header(name, 0x08);
+        oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00));
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and double value @a value
+    */
+    void write_bson_double(const string_t& name,
+                           const double value)
+    {
+        write_bson_entry_header(name, 0x01);
+        write_number<double, true>(value);
+    }
+
+    /*!
+    @return The size of the BSON-encoded string in @a value
+    */
+    static std::size_t calc_bson_string_size(const string_t& value)
+    {
+        return sizeof(std::int32_t) + value.size() + 1ul;
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and string value @a value
+    */
+    void write_bson_string(const string_t& name,
+                           const string_t& value)
+    {
+        write_bson_entry_header(name, 0x02);
+
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul));
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(value.c_str()),
+            value.size() + 1);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and null value
+    */
+    void write_bson_null(const string_t& name)
+    {
+        write_bson_entry_header(name, 0x0A);
+    }
+
+    /*!
+    @return The size of the BSON-encoded integer @a value
+    */
+    static std::size_t calc_bson_integer_size(const std::int64_t value)
+    {
+        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
+        {
+            return sizeof(std::int32_t);
+        }
+        else
+        {
+            return sizeof(std::int64_t);
+        }
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and integer @a value
+    */
+    void write_bson_integer(const string_t& name,
+                            const std::int64_t value)
+    {
+        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
+        {
+            write_bson_entry_header(name, 0x10); // int32
+            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
+        }
+        else
+        {
+            write_bson_entry_header(name, 0x12); // int64
+            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
+        }
+    }
+
+    /*!
+    @return The size of the BSON-encoded unsigned integer in @a j
+    */
+    static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept
+    {
+        return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+               ? sizeof(std::int32_t)
+               : sizeof(std::int64_t);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and unsigned @a value
+    */
+    void write_bson_unsigned(const string_t& name,
+                             const std::uint64_t value)
+    {
+        if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+        {
+            write_bson_entry_header(name, 0x10 /* int32 */);
+            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
+        }
+        else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+        {
+            write_bson_entry_header(name, 0x12 /* int64 */);
+            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
+        }
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64"));
+        }
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and object @a value
+    */
+    void write_bson_object_entry(const string_t& name,
+                                 const typename BasicJsonType::object_t& value)
+    {
+        write_bson_entry_header(name, 0x03); // object
+        write_bson_object(value);
+    }
+
+    /*!
+    @return The size of the BSON-encoded array @a value
+    */
+    static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value)
+    {
+        std::size_t embedded_document_size = 0ul;
+        std::size_t array_index = 0ul;
+
+        for (const auto& el : value)
+        {
+            embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el);
+        }
+
+        return sizeof(std::int32_t) + embedded_document_size + 1ul;
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and array @a value
+    */
+    void write_bson_array(const string_t& name,
+                          const typename BasicJsonType::array_t& value)
+    {
+        write_bson_entry_header(name, 0x04); // array
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value)));
+
+        std::size_t array_index = 0ul;
+
+        for (const auto& el : value)
+        {
+            write_bson_element(std::to_string(array_index++), el);
+        }
+
+        oa->write_character(to_char_type(0x00));
+    }
+
+    /*!
+    @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
+    @return The calculated size for the BSON document entry for @a j with the given @a name.
+    */
+    static std::size_t calc_bson_element_size(const string_t& name,
+            const BasicJsonType& j)
+    {
+        const auto header_size = calc_bson_entry_header_size(name);
+        switch (j.type())
+        {
+            case value_t::object:
+                return header_size + calc_bson_object_size(*j.m_value.object);
+
+            case value_t::array:
+                return header_size + calc_bson_array_size(*j.m_value.array);
+
+            case value_t::boolean:
+                return header_size + 1ul;
+
+            case value_t::number_float:
+                return header_size + 8ul;
+
+            case value_t::number_integer:
+                return header_size + calc_bson_integer_size(j.m_value.number_integer);
+
+            case value_t::number_unsigned:
+                return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned);
+
+            case value_t::string:
+                return header_size + calc_bson_string_size(*j.m_value.string);
+
+            case value_t::null:
+                return header_size + 0ul;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                return 0ul;
+                // LCOV_EXCL_STOP
+        };
+    }
+
+    /*!
+    @brief Serializes the JSON value @a j to BSON and associates it with the
+           key @a name.
+    @param name The name to associate with the JSON entity @a j within the
+                current BSON document
+    @return The size of the BSON entry
+    */
+    void write_bson_element(const string_t& name,
+                            const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::object:
+                return write_bson_object_entry(name, *j.m_value.object);
+
+            case value_t::array:
+                return write_bson_array(name, *j.m_value.array);
+
+            case value_t::boolean:
+                return write_bson_boolean(name, j.m_value.boolean);
+
+            case value_t::number_float:
+                return write_bson_double(name, j.m_value.number_float);
+
+            case value_t::number_integer:
+                return write_bson_integer(name, j.m_value.number_integer);
+
+            case value_t::number_unsigned:
+                return write_bson_unsigned(name, j.m_value.number_unsigned);
+
+            case value_t::string:
+                return write_bson_string(name, *j.m_value.string);
+
+            case value_t::null:
+                return write_bson_null(name);
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                return;
+                // LCOV_EXCL_STOP
+        };
+    }
+
+    /*!
+    @brief Calculates the size of the BSON serialization of the given
+           JSON-object @a j.
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value)
+    {
+        std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul,
+                                    [](size_t result, const typename BasicJsonType::object_t::value_type & el)
+        {
+            return result += calc_bson_element_size(el.first, el.second);
+        });
+
+        return sizeof(std::int32_t) + document_size + 1ul;
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    void write_bson_object(const typename BasicJsonType::object_t& value)
+    {
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value)));
+
+        for (const auto& el : value)
+        {
+            write_bson_element(el.first, el.second);
+        }
+
+        oa->write_character(to_char_type(0x00));
+    }
+
+    //////////
+    // CBOR //
+    //////////
+
+    static constexpr CharType get_cbor_float_prefix(float /*unused*/)
+    {
+        return to_char_type(0xFA);  // Single-Precision Float
+    }
+
+    static constexpr CharType get_cbor_float_prefix(double /*unused*/)
+    {
+        return to_char_type(0xFB);  // Double-Precision Float
+    }
+
+    /////////////
+    // MsgPack //
+    /////////////
+
+    static constexpr CharType get_msgpack_float_prefix(float /*unused*/)
+    {
+        return to_char_type(0xCA);  // float 32
+    }
+
+    static constexpr CharType get_msgpack_float_prefix(double /*unused*/)
+    {
+        return to_char_type(0xCB);  // float 64
+    }
+
+    ////////////
+    // UBJSON //
+    ////////////
+
+    // UBJSON: write number (floating point)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_floating_point<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if (add_prefix)
+        {
+            oa->write_character(get_ubjson_float_prefix(n));
+        }
+        write_number(n);
+    }
+
+    // UBJSON: write number (unsigned integer)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_unsigned<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('i'));  // int8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if (n <= (std::numeric_limits<uint8_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('U'));  // uint8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('I'));  // int16
+            }
+            write_number(static_cast<int16_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('l'));  // int32
+            }
+            write_number(static_cast<int32_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('L'));  // int64
+            }
+            write_number(static_cast<int64_t>(n));
+        }
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
+        }
+    }
+
+    // UBJSON: write number (signed integer)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_signed<NumberType>::value and
+                 not std::is_floating_point<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('i'));  // int8
+            }
+            write_number(static_cast<int8_t>(n));
+        }
+        else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('U'));  // uint8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('I'));  // int16
+            }
+            write_number(static_cast<int16_t>(n));
+        }
+        else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('l'));  // int32
+            }
+            write_number(static_cast<int32_t>(n));
+        }
+        else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('L'));  // int64
+            }
+            write_number(static_cast<int64_t>(n));
+        }
+        // LCOV_EXCL_START
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
+        }
+        // LCOV_EXCL_STOP
+    }
+
+    /*!
+    @brief determine the type prefix of container values
+
+    @note This function does not need to be 100% accurate when it comes to
+          integer limits. In case a number exceeds the limits of int64_t,
+          this will be detected by a later call to function
+          write_number_with_ubjson_prefix. Therefore, we return 'L' for any
+          value that does not fit the previous limits.
+    */
+    CharType ubjson_prefix(const BasicJsonType& j) const noexcept
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+                return 'Z';
+
+            case value_t::boolean:
+                return j.m_value.boolean ? 'T' : 'F';
+
+            case value_t::number_integer:
+            {
+                if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                {
+                    return 'i';
+                }
+                if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    return 'U';
+                }
+                if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                {
+                    return 'I';
+                }
+                if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                {
+                    return 'l';
+                }
+                // no check and assume int64_t (see note above)
+                return 'L';
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)())
+                {
+                    return 'i';
+                }
+                if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    return 'U';
+                }
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)())
+                {
+                    return 'I';
+                }
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)())
+                {
+                    return 'l';
+                }
+                // no check and assume int64_t (see note above)
+                return 'L';
+            }
+
+            case value_t::number_float:
+                return get_ubjson_float_prefix(j.m_value.number_float);
+
+            case value_t::string:
+                return 'S';
+
+            case value_t::array:
+                return '[';
+
+            case value_t::object:
+                return '{';
+
+            default:  // discarded values
+                return 'N';
+        }
+    }
+
+    static constexpr CharType get_ubjson_float_prefix(float /*unused*/)
+    {
+        return 'd';  // float 32
+    }
+
+    static constexpr CharType get_ubjson_float_prefix(double /*unused*/)
+    {
+        return 'D';  // float 64
+    }
+
+    ///////////////////////
+    // Utility functions //
+    ///////////////////////
+
+    /*
+    @brief write a number to output input
+    @param[in] n number of type @a NumberType
+    @tparam NumberType the type of the number
+    @tparam OutputIsLittleEndian Set to true if output data is
+                                 required to be little endian
+
+    @note This function needs to respect the system's endianess, because bytes
+          in CBOR, MessagePack, and UBJSON are stored in network order (big
+          endian) and therefore need reordering on little endian systems.
+    */
+    template<typename NumberType, bool OutputIsLittleEndian = false>
+    void write_number(const NumberType n)
+    {
+        // step 1: write number to array of length NumberType
+        std::array<CharType, sizeof(NumberType)> vec;
+        std::memcpy(vec.data(), &n, sizeof(NumberType));
+
+        // step 2: write array to output (with possible reordering)
+        if (is_little_endian and not OutputIsLittleEndian)
+        {
+            // reverse byte order prior to conversion if necessary
+            std::reverse(vec.begin(), vec.end());
+        }
+
+        oa->write_characters(vec.data(), sizeof(NumberType));
+    }
+
+  public:
+    // The following to_char_type functions are implement the conversion
+    // between uint8_t and CharType. In case CharType is not unsigned,
+    // such a conversion is required to allow values greater than 128.
+    // See <https://github.com/nlohmann/json/issues/1286> for a discussion.
+    template < typename C = CharType,
+               enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr >
+    static constexpr CharType to_char_type(std::uint8_t x) noexcept
+    {
+        return *reinterpret_cast<char*>(&x);
+    }
+
+    template < typename C = CharType,
+               enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr >
+    static CharType to_char_type(std::uint8_t x) noexcept
+    {
+        static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t");
+        static_assert(std::is_pod<CharType>::value, "CharType must be POD");
+        CharType result;
+        std::memcpy(&result, &x, sizeof(x));
+        return result;
+    }
+
+    template<typename C = CharType,
+             enable_if_t<std::is_unsigned<C>::value>* = nullptr>
+    static constexpr CharType to_char_type(std::uint8_t x) noexcept
+    {
+        return x;
+    }
+
+    template < typename InputCharType, typename C = CharType,
+               enable_if_t <
+                   std::is_signed<C>::value and
+                   std::is_signed<char>::value and
+                   std::is_same<char, typename std::remove_cv<InputCharType>::type>::value
+                   > * = nullptr >
+    static constexpr CharType to_char_type(InputCharType x) noexcept
+    {
+        return x;
+    }
+
+  private:
+    /// whether we can assume little endianess
+    const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
+
+    /// the output
+    output_adapter_t<CharType> oa = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/serializer.hpp>
+
+
+#include <algorithm> // reverse, remove, fill, find, none_of
+#include <array> // array
+#include <cassert> // assert
+#include <ciso646> // and, or
+#include <clocale> // localeconv, lconv
+#include <cmath> // labs, isfinite, isnan, signbit
+#include <cstddef> // size_t, ptrdiff_t
+#include <cstdint> // uint8_t
+#include <cstdio> // snprintf
+#include <limits> // numeric_limits
+#include <string> // string
+#include <type_traits> // is_same
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/conversions/to_chars.hpp>
+
+
+#include <cassert> // assert
+#include <ciso646> // or, and, not
+#include <cmath>   // signbit, isfinite
+#include <cstdint> // intN_t, uintN_t
+#include <cstring> // memcpy, memmove
+
+namespace nlohmann
+{
+namespace detail
+{
+
+/*!
+@brief implements the Grisu2 algorithm for binary to decimal floating-point
+conversion.
+
+This implementation is a slightly modified version of the reference
+implementation which may be obtained from
+http://florian.loitsch.com/publications (bench.tar.gz).
+
+The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch.
+
+For a detailed description of the algorithm see:
+
+[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with
+    Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming
+    Language Design and Implementation, PLDI 2010
+[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately",
+    Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language
+    Design and Implementation, PLDI 1996
+*/
+namespace dtoa_impl
+{
+
+template <typename Target, typename Source>
+Target reinterpret_bits(const Source source)
+{
+    static_assert(sizeof(Target) == sizeof(Source), "size mismatch");
+
+    Target target;
+    std::memcpy(&target, &source, sizeof(Source));
+    return target;
+}
+
+struct diyfp // f * 2^e
+{
+    static constexpr int kPrecision = 64; // = q
+
+    uint64_t f = 0;
+    int e = 0;
+
+    constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {}
+
+    /*!
+    @brief returns x - y
+    @pre x.e == y.e and x.f >= y.f
+    */
+    static diyfp sub(const diyfp& x, const diyfp& y) noexcept
+    {
+        assert(x.e == y.e);
+        assert(x.f >= y.f);
+
+        return {x.f - y.f, x.e};
+    }
+
+    /*!
+    @brief returns x * y
+    @note The result is rounded. (Only the upper q bits are returned.)
+    */
+    static diyfp mul(const diyfp& x, const diyfp& y) noexcept
+    {
+        static_assert(kPrecision == 64, "internal error");
+
+        // Computes:
+        //  f = round((x.f * y.f) / 2^q)
+        //  e = x.e + y.e + q
+
+        // Emulate the 64-bit * 64-bit multiplication:
+        //
+        // p = u * v
+        //   = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi)
+        //   = (u_lo v_lo         ) + 2^32 ((u_lo v_hi         ) + (u_hi v_lo         )) + 2^64 (u_hi v_hi         )
+        //   = (p0                ) + 2^32 ((p1                ) + (p2                )) + 2^64 (p3                )
+        //   = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3                )
+        //   = (p0_lo             ) + 2^32 (p0_hi + p1_lo + p2_lo                      ) + 2^64 (p1_hi + p2_hi + p3)
+        //   = (p0_lo             ) + 2^32 (Q                                          ) + 2^64 (H                 )
+        //   = (p0_lo             ) + 2^32 (Q_lo + 2^32 Q_hi                           ) + 2^64 (H                 )
+        //
+        // (Since Q might be larger than 2^32 - 1)
+        //
+        //   = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H)
+        //
+        // (Q_hi + H does not overflow a 64-bit int)
+        //
+        //   = p_lo + 2^64 p_hi
+
+        const uint64_t u_lo = x.f & 0xFFFFFFFF;
+        const uint64_t u_hi = x.f >> 32;
+        const uint64_t v_lo = y.f & 0xFFFFFFFF;
+        const uint64_t v_hi = y.f >> 32;
+
+        const uint64_t p0 = u_lo * v_lo;
+        const uint64_t p1 = u_lo * v_hi;
+        const uint64_t p2 = u_hi * v_lo;
+        const uint64_t p3 = u_hi * v_hi;
+
+        const uint64_t p0_hi = p0 >> 32;
+        const uint64_t p1_lo = p1 & 0xFFFFFFFF;
+        const uint64_t p1_hi = p1 >> 32;
+        const uint64_t p2_lo = p2 & 0xFFFFFFFF;
+        const uint64_t p2_hi = p2 >> 32;
+
+        uint64_t Q = p0_hi + p1_lo + p2_lo;
+
+        // The full product might now be computed as
+        //
+        // p_hi = p3 + p2_hi + p1_hi + (Q >> 32)
+        // p_lo = p0_lo + (Q << 32)
+        //
+        // But in this particular case here, the full p_lo is not required.
+        // Effectively we only need to add the highest bit in p_lo to p_hi (and
+        // Q_hi + 1 does not overflow).
+
+        Q += uint64_t{1} << (64 - 32 - 1); // round, ties up
+
+        const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32);
+
+        return {h, x.e + y.e + 64};
+    }
+
+    /*!
+    @brief normalize x such that the significand is >= 2^(q-1)
+    @pre x.f != 0
+    */
+    static diyfp normalize(diyfp x) noexcept
+    {
+        assert(x.f != 0);
+
+        while ((x.f >> 63) == 0)
+        {
+            x.f <<= 1;
+            x.e--;
+        }
+
+        return x;
+    }
+
+    /*!
+    @brief normalize x such that the result has the exponent E
+    @pre e >= x.e and the upper e - x.e bits of x.f must be zero.
+    */
+    static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept
+    {
+        const int delta = x.e - target_exponent;
+
+        assert(delta >= 0);
+        assert(((x.f << delta) >> delta) == x.f);
+
+        return {x.f << delta, target_exponent};
+    }
+};
+
+struct boundaries
+{
+    diyfp w;
+    diyfp minus;
+    diyfp plus;
+};
+
+/*!
+Compute the (normalized) diyfp representing the input number 'value' and its
+boundaries.
+
+@pre value must be finite and positive
+*/
+template <typename FloatType>
+boundaries compute_boundaries(FloatType value)
+{
+    assert(std::isfinite(value));
+    assert(value > 0);
+
+    // Convert the IEEE representation into a diyfp.
+    //
+    // If v is denormal:
+    //      value = 0.F * 2^(1 - bias) = (          F) * 2^(1 - bias - (p-1))
+    // If v is normalized:
+    //      value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1))
+
+    static_assert(std::numeric_limits<FloatType>::is_iec559,
+                  "internal error: dtoa_short requires an IEEE-754 floating-point implementation");
+
+    constexpr int      kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit)
+    constexpr int      kBias      = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1);
+    constexpr int      kMinExp    = 1 - kBias;
+    constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1)
+
+    using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type;
+
+    const uint64_t bits = reinterpret_bits<bits_type>(value);
+    const uint64_t E = bits >> (kPrecision - 1);
+    const uint64_t F = bits & (kHiddenBit - 1);
+
+    const bool is_denormal = (E == 0);
+    const diyfp v = is_denormal
+                    ? diyfp(F, kMinExp)
+                    : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias);
+
+    // Compute the boundaries m- and m+ of the floating-point value
+    // v = f * 2^e.
+    //
+    // Determine v- and v+, the floating-point predecessor and successor if v,
+    // respectively.
+    //
+    //      v- = v - 2^e        if f != 2^(p-1) or e == e_min                (A)
+    //         = v - 2^(e-1)    if f == 2^(p-1) and e > e_min                (B)
+    //
+    //      v+ = v + 2^e
+    //
+    // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_
+    // between m- and m+ round to v, regardless of how the input rounding
+    // algorithm breaks ties.
+    //
+    //      ---+-------------+-------------+-------------+-------------+---  (A)
+    //         v-            m-            v             m+            v+
+    //
+    //      -----------------+------+------+-------------+-------------+---  (B)
+    //                       v-     m-     v             m+            v+
+
+    const bool lower_boundary_is_closer = (F == 0 and E > 1);
+    const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1);
+    const diyfp m_minus = lower_boundary_is_closer
+                          ? diyfp(4 * v.f - 1, v.e - 2)  // (B)
+                          : diyfp(2 * v.f - 1, v.e - 1); // (A)
+
+    // Determine the normalized w+ = m+.
+    const diyfp w_plus = diyfp::normalize(m_plus);
+
+    // Determine w- = m- such that e_(w-) = e_(w+).
+    const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e);
+
+    return {diyfp::normalize(v), w_minus, w_plus};
+}
+
+// Given normalized diyfp w, Grisu needs to find a (normalized) cached
+// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies
+// within a certain range [alpha, gamma] (Definition 3.2 from [1])
+//
+//      alpha <= e = e_c + e_w + q <= gamma
+//
+// or
+//
+//      f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q
+//                          <= f_c * f_w * 2^gamma
+//
+// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies
+//
+//      2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma
+//
+// or
+//
+//      2^(q - 2 + alpha) <= c * w < 2^(q + gamma)
+//
+// The choice of (alpha,gamma) determines the size of the table and the form of
+// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well
+// in practice:
+//
+// The idea is to cut the number c * w = f * 2^e into two parts, which can be
+// processed independently: An integral part p1, and a fractional part p2:
+//
+//      f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e
+//              = (f div 2^-e) + (f mod 2^-e) * 2^e
+//              = p1 + p2 * 2^e
+//
+// The conversion of p1 into decimal form requires a series of divisions and
+// modulos by (a power of) 10. These operations are faster for 32-bit than for
+// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be
+// achieved by choosing
+//
+//      -e >= 32   or   e <= -32 := gamma
+//
+// In order to convert the fractional part
+//
+//      p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ...
+//
+// into decimal form, the fraction is repeatedly multiplied by 10 and the digits
+// d[-i] are extracted in order:
+//
+//      (10 * p2) div 2^-e = d[-1]
+//      (10 * p2) mod 2^-e = d[-2] / 10^1 + ...
+//
+// The multiplication by 10 must not overflow. It is sufficient to choose
+//
+//      10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64.
+//
+// Since p2 = f mod 2^-e < 2^-e,
+//
+//      -e <= 60   or   e >= -60 := alpha
+
+constexpr int kAlpha = -60;
+constexpr int kGamma = -32;
+
+struct cached_power // c = f * 2^e ~= 10^k
+{
+    uint64_t f;
+    int e;
+    int k;
+};
+
+/*!
+For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached
+power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c
+satisfies (Definition 3.2 from [1])
+
+     alpha <= e_c + e + q <= gamma.
+*/
+inline cached_power get_cached_power_for_binary_exponent(int e)
+{
+    // Now
+    //
+    //      alpha <= e_c + e + q <= gamma                                    (1)
+    //      ==> f_c * 2^alpha <= c * 2^e * 2^q
+    //
+    // and since the c's are normalized, 2^(q-1) <= f_c,
+    //
+    //      ==> 2^(q - 1 + alpha) <= c * 2^(e + q)
+    //      ==> 2^(alpha - e - 1) <= c
+    //
+    // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as
+    //
+    //      k = ceil( log_10( 2^(alpha - e - 1) ) )
+    //        = ceil( (alpha - e - 1) * log_10(2) )
+    //
+    // From the paper:
+    // "In theory the result of the procedure could be wrong since c is rounded,
+    //  and the computation itself is approximated [...]. In practice, however,
+    //  this simple function is sufficient."
+    //
+    // For IEEE double precision floating-point numbers converted into
+    // normalized diyfp's w = f * 2^e, with q = 64,
+    //
+    //      e >= -1022      (min IEEE exponent)
+    //           -52        (p - 1)
+    //           -52        (p - 1, possibly normalize denormal IEEE numbers)
+    //           -11        (normalize the diyfp)
+    //         = -1137
+    //
+    // and
+    //
+    //      e <= +1023      (max IEEE exponent)
+    //           -52        (p - 1)
+    //           -11        (normalize the diyfp)
+    //         = 960
+    //
+    // This binary exponent range [-1137,960] results in a decimal exponent
+    // range [-307,324]. One does not need to store a cached power for each
+    // k in this range. For each such k it suffices to find a cached power
+    // such that the exponent of the product lies in [alpha,gamma].
+    // This implies that the difference of the decimal exponents of adjacent
+    // table entries must be less than or equal to
+    //
+    //      floor( (gamma - alpha) * log_10(2) ) = 8.
+    //
+    // (A smaller distance gamma-alpha would require a larger table.)
+
+    // NB:
+    // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34.
+
+    constexpr int kCachedPowersSize = 79;
+    constexpr int kCachedPowersMinDecExp = -300;
+    constexpr int kCachedPowersDecStep = 8;
+
+    static constexpr cached_power kCachedPowers[] =
+    {
+        { 0xAB70FE17C79AC6CA, -1060, -300 },
+        { 0xFF77B1FCBEBCDC4F, -1034, -292 },
+        { 0xBE5691EF416BD60C, -1007, -284 },
+        { 0x8DD01FAD907FFC3C,  -980, -276 },
+        { 0xD3515C2831559A83,  -954, -268 },
+        { 0x9D71AC8FADA6C9B5,  -927, -260 },
+        { 0xEA9C227723EE8BCB,  -901, -252 },
+        { 0xAECC49914078536D,  -874, -244 },
+        { 0x823C12795DB6CE57,  -847, -236 },
+        { 0xC21094364DFB5637,  -821, -228 },
+        { 0x9096EA6F3848984F,  -794, -220 },
+        { 0xD77485CB25823AC7,  -768, -212 },
+        { 0xA086CFCD97BF97F4,  -741, -204 },
+        { 0xEF340A98172AACE5,  -715, -196 },
+        { 0xB23867FB2A35B28E,  -688, -188 },
+        { 0x84C8D4DFD2C63F3B,  -661, -180 },
+        { 0xC5DD44271AD3CDBA,  -635, -172 },
+        { 0x936B9FCEBB25C996,  -608, -164 },
+        { 0xDBAC6C247D62A584,  -582, -156 },
+        { 0xA3AB66580D5FDAF6,  -555, -148 },
+        { 0xF3E2F893DEC3F126,  -529, -140 },
+        { 0xB5B5ADA8AAFF80B8,  -502, -132 },
+        { 0x87625F056C7C4A8B,  -475, -124 },
+        { 0xC9BCFF6034C13053,  -449, -116 },
+        { 0x964E858C91BA2655,  -422, -108 },
+        { 0xDFF9772470297EBD,  -396, -100 },
+        { 0xA6DFBD9FB8E5B88F,  -369,  -92 },
+        { 0xF8A95FCF88747D94,  -343,  -84 },
+        { 0xB94470938FA89BCF,  -316,  -76 },
+        { 0x8A08F0F8BF0F156B,  -289,  -68 },
+        { 0xCDB02555653131B6,  -263,  -60 },
+        { 0x993FE2C6D07B7FAC,  -236,  -52 },
+        { 0xE45C10C42A2B3B06,  -210,  -44 },
+        { 0xAA242499697392D3,  -183,  -36 },
+        { 0xFD87B5F28300CA0E,  -157,  -28 },
+        { 0xBCE5086492111AEB,  -130,  -20 },
+        { 0x8CBCCC096F5088CC,  -103,  -12 },
+        { 0xD1B71758E219652C,   -77,   -4 },
+        { 0x9C40000000000000,   -50,    4 },
+        { 0xE8D4A51000000000,   -24,   12 },
+        { 0xAD78EBC5AC620000,     3,   20 },
+        { 0x813F3978F8940984,    30,   28 },
+        { 0xC097CE7BC90715B3,    56,   36 },
+        { 0x8F7E32CE7BEA5C70,    83,   44 },
+        { 0xD5D238A4ABE98068,   109,   52 },
+        { 0x9F4F2726179A2245,   136,   60 },
+        { 0xED63A231D4C4FB27,   162,   68 },
+        { 0xB0DE65388CC8ADA8,   189,   76 },
+        { 0x83C7088E1AAB65DB,   216,   84 },
+        { 0xC45D1DF942711D9A,   242,   92 },
+        { 0x924D692CA61BE758,   269,  100 },
+        { 0xDA01EE641A708DEA,   295,  108 },
+        { 0xA26DA3999AEF774A,   322,  116 },
+        { 0xF209787BB47D6B85,   348,  124 },
+        { 0xB454E4A179DD1877,   375,  132 },
+        { 0x865B86925B9BC5C2,   402,  140 },
+        { 0xC83553C5C8965D3D,   428,  148 },
+        { 0x952AB45CFA97A0B3,   455,  156 },
+        { 0xDE469FBD99A05FE3,   481,  164 },
+        { 0xA59BC234DB398C25,   508,  172 },
+        { 0xF6C69A72A3989F5C,   534,  180 },
+        { 0xB7DCBF5354E9BECE,   561,  188 },
+        { 0x88FCF317F22241E2,   588,  196 },
+        { 0xCC20CE9BD35C78A5,   614,  204 },
+        { 0x98165AF37B2153DF,   641,  212 },
+        { 0xE2A0B5DC971F303A,   667,  220 },
+        { 0xA8D9D1535CE3B396,   694,  228 },
+        { 0xFB9B7CD9A4A7443C,   720,  236 },
+        { 0xBB764C4CA7A44410,   747,  244 },
+        { 0x8BAB8EEFB6409C1A,   774,  252 },
+        { 0xD01FEF10A657842C,   800,  260 },
+        { 0x9B10A4E5E9913129,   827,  268 },
+        { 0xE7109BFBA19C0C9D,   853,  276 },
+        { 0xAC2820D9623BF429,   880,  284 },
+        { 0x80444B5E7AA7CF85,   907,  292 },
+        { 0xBF21E44003ACDD2D,   933,  300 },
+        { 0x8E679C2F5E44FF8F,   960,  308 },
+        { 0xD433179D9C8CB841,   986,  316 },
+        { 0x9E19DB92B4E31BA9,  1013,  324 },
+    };
+
+    // This computation gives exactly the same results for k as
+    //      k = ceil((kAlpha - e - 1) * 0.30102999566398114)
+    // for |e| <= 1500, but doesn't require floating-point operations.
+    // NB: log_10(2) ~= 78913 / 2^18
+    assert(e >= -1500);
+    assert(e <=  1500);
+    const int f = kAlpha - e - 1;
+    const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0);
+
+    const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep;
+    assert(index >= 0);
+    assert(index < kCachedPowersSize);
+    static_cast<void>(kCachedPowersSize); // Fix warning.
+
+    const cached_power cached = kCachedPowers[index];
+    assert(kAlpha <= cached.e + e + 64);
+    assert(kGamma >= cached.e + e + 64);
+
+    return cached;
+}
+
+/*!
+For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k.
+For n == 0, returns 1 and sets pow10 := 1.
+*/
+inline int find_largest_pow10(const uint32_t n, uint32_t& pow10)
+{
+    // LCOV_EXCL_START
+    if (n >= 1000000000)
+    {
+        pow10 = 1000000000;
+        return 10;
+    }
+    // LCOV_EXCL_STOP
+    else if (n >= 100000000)
+    {
+        pow10 = 100000000;
+        return  9;
+    }
+    else if (n >= 10000000)
+    {
+        pow10 = 10000000;
+        return  8;
+    }
+    else if (n >= 1000000)
+    {
+        pow10 = 1000000;
+        return  7;
+    }
+    else if (n >= 100000)
+    {
+        pow10 = 100000;
+        return  6;
+    }
+    else if (n >= 10000)
+    {
+        pow10 = 10000;
+        return  5;
+    }
+    else if (n >= 1000)
+    {
+        pow10 = 1000;
+        return  4;
+    }
+    else if (n >= 100)
+    {
+        pow10 = 100;
+        return  3;
+    }
+    else if (n >= 10)
+    {
+        pow10 = 10;
+        return  2;
+    }
+    else
+    {
+        pow10 = 1;
+        return 1;
+    }
+}
+
+inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta,
+                         uint64_t rest, uint64_t ten_k)
+{
+    assert(len >= 1);
+    assert(dist <= delta);
+    assert(rest <= delta);
+    assert(ten_k > 0);
+
+    //               <--------------------------- delta ---->
+    //                                  <---- dist --------->
+    // --------------[------------------+-------------------]--------------
+    //               M-                 w                   M+
+    //
+    //                                  ten_k
+    //                                <------>
+    //                                       <---- rest ---->
+    // --------------[------------------+----+--------------]--------------
+    //                                  w    V
+    //                                       = buf * 10^k
+    //
+    // ten_k represents a unit-in-the-last-place in the decimal representation
+    // stored in buf.
+    // Decrement buf by ten_k while this takes buf closer to w.
+
+    // The tests are written in this order to avoid overflow in unsigned
+    // integer arithmetic.
+
+    while (rest < dist
+            and delta - rest >= ten_k
+            and (rest + ten_k < dist or dist - rest > rest + ten_k - dist))
+    {
+        assert(buf[len - 1] != '0');
+        buf[len - 1]--;
+        rest += ten_k;
+    }
+}
+
+/*!
+Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+.
+M- and M+ must be normalized and share the same exponent -60 <= e <= -32.
+*/
+inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent,
+                             diyfp M_minus, diyfp w, diyfp M_plus)
+{
+    static_assert(kAlpha >= -60, "internal error");
+    static_assert(kGamma <= -32, "internal error");
+
+    // Generates the digits (and the exponent) of a decimal floating-point
+    // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's
+    // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma.
+    //
+    //               <--------------------------- delta ---->
+    //                                  <---- dist --------->
+    // --------------[------------------+-------------------]--------------
+    //               M-                 w                   M+
+    //
+    // Grisu2 generates the digits of M+ from left to right and stops as soon as
+    // V is in [M-,M+].
+
+    assert(M_plus.e >= kAlpha);
+    assert(M_plus.e <= kGamma);
+
+    uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e)
+    uint64_t dist  = diyfp::sub(M_plus, w      ).f; // (significand of (M+ - w ), implicit exponent is e)
+
+    // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0):
+    //
+    //      M+ = f * 2^e
+    //         = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e
+    //         = ((p1        ) * 2^-e + (p2        )) * 2^e
+    //         = p1 + p2 * 2^e
+
+    const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e);
+
+    auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.)
+    uint64_t p2 = M_plus.f & (one.f - 1);                    // p2 = f mod 2^-e
+
+    // 1)
+    //
+    // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0]
+
+    assert(p1 > 0);
+
+    uint32_t pow10;
+    const int k = find_largest_pow10(p1, pow10);
+
+    //      10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1)
+    //
+    //      p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1))
+    //         = (d[k-1]         ) * 10^(k-1) + (p1 mod 10^(k-1))
+    //
+    //      M+ = p1                                             + p2 * 2^e
+    //         = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1))          + p2 * 2^e
+    //         = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e
+    //         = d[k-1] * 10^(k-1) + (                         rest) * 2^e
+    //
+    // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0)
+    //
+    //      p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0]
+    //
+    // but stop as soon as
+    //
+    //      rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e
+
+    int n = k;
+    while (n > 0)
+    {
+        // Invariants:
+        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)    (buffer = 0 for n = k)
+        //      pow10 = 10^(n-1) <= p1 < 10^n
+        //
+        const uint32_t d = p1 / pow10;  // d = p1 div 10^(n-1)
+        const uint32_t r = p1 % pow10;  // r = p1 mod 10^(n-1)
+        //
+        //      M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e
+        //         = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e)
+        //
+        assert(d <= 9);
+        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+        //
+        //      M+ = buffer * 10^(n-1) + (r + p2 * 2^e)
+        //
+        p1 = r;
+        n--;
+        //
+        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)
+        //      pow10 = 10^n
+        //
+
+        // Now check if enough digits have been generated.
+        // Compute
+        //
+        //      p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e
+        //
+        // Note:
+        // Since rest and delta share the same exponent e, it suffices to
+        // compare the significands.
+        const uint64_t rest = (uint64_t{p1} << -one.e) + p2;
+        if (rest <= delta)
+        {
+            // V = buffer * 10^n, with M- <= V <= M+.
+
+            decimal_exponent += n;
+
+            // We may now just stop. But instead look if the buffer could be
+            // decremented to bring V closer to w.
+            //
+            // pow10 = 10^n is now 1 ulp in the decimal representation V.
+            // The rounding procedure works with diyfp's with an implicit
+            // exponent of e.
+            //
+            //      10^n = (10^n * 2^-e) * 2^e = ulp * 2^e
+            //
+            const uint64_t ten_n = uint64_t{pow10} << -one.e;
+            grisu2_round(buffer, length, dist, delta, rest, ten_n);
+
+            return;
+        }
+
+        pow10 /= 10;
+        //
+        //      pow10 = 10^(n-1) <= p1 < 10^n
+        // Invariants restored.
+    }
+
+    // 2)
+    //
+    // The digits of the integral part have been generated:
+    //
+    //      M+ = d[k-1]...d[1]d[0] + p2 * 2^e
+    //         = buffer            + p2 * 2^e
+    //
+    // Now generate the digits of the fractional part p2 * 2^e.
+    //
+    // Note:
+    // No decimal point is generated: the exponent is adjusted instead.
+    //
+    // p2 actually represents the fraction
+    //
+    //      p2 * 2^e
+    //          = p2 / 2^-e
+    //          = d[-1] / 10^1 + d[-2] / 10^2 + ...
+    //
+    // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...)
+    //
+    //      p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m
+    //                      + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...)
+    //
+    // using
+    //
+    //      10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e)
+    //                = (                   d) * 2^-e + (                   r)
+    //
+    // or
+    //      10^m * p2 * 2^e = d + r * 2^e
+    //
+    // i.e.
+    //
+    //      M+ = buffer + p2 * 2^e
+    //         = buffer + 10^-m * (d + r * 2^e)
+    //         = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e
+    //
+    // and stop as soon as 10^-m * r * 2^e <= delta * 2^e
+
+    assert(p2 > delta);
+
+    int m = 0;
+    for (;;)
+    {
+        // Invariant:
+        //      M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e
+        //         = buffer * 10^-m + 10^-m * (p2                                 ) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * (10 * p2)                   ) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e
+        //
+        assert(p2 <= UINT64_MAX / 10);
+        p2 *= 10;
+        const uint64_t d = p2 >> -one.e;     // d = (10 * p2) div 2^-e
+        const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e
+        //
+        //      M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e))
+        //         = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e
+        //
+        assert(d <= 9);
+        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+        //
+        //      M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e
+        //
+        p2 = r;
+        m++;
+        //
+        //      M+ = buffer * 10^-m + 10^-m * p2 * 2^e
+        // Invariant restored.
+
+        // Check if enough digits have been generated.
+        //
+        //      10^-m * p2 * 2^e <= delta * 2^e
+        //              p2 * 2^e <= 10^m * delta * 2^e
+        //                    p2 <= 10^m * delta
+        delta *= 10;
+        dist  *= 10;
+        if (p2 <= delta)
+        {
+            break;
+        }
+    }
+
+    // V = buffer * 10^-m, with M- <= V <= M+.
+
+    decimal_exponent -= m;
+
+    // 1 ulp in the decimal representation is now 10^-m.
+    // Since delta and dist are now scaled by 10^m, we need to do the
+    // same with ulp in order to keep the units in sync.
+    //
+    //      10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e
+    //
+    const uint64_t ten_m = one.f;
+    grisu2_round(buffer, length, dist, delta, p2, ten_m);
+
+    // By construction this algorithm generates the shortest possible decimal
+    // number (Loitsch, Theorem 6.2) which rounds back to w.
+    // For an input number of precision p, at least
+    //
+    //      N = 1 + ceil(p * log_10(2))
+    //
+    // decimal digits are sufficient to identify all binary floating-point
+    // numbers (Matula, "In-and-Out conversions").
+    // This implies that the algorithm does not produce more than N decimal
+    // digits.
+    //
+    //      N = 17 for p = 53 (IEEE double precision)
+    //      N = 9  for p = 24 (IEEE single precision)
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+inline void grisu2(char* buf, int& len, int& decimal_exponent,
+                   diyfp m_minus, diyfp v, diyfp m_plus)
+{
+    assert(m_plus.e == m_minus.e);
+    assert(m_plus.e == v.e);
+
+    //  --------(-----------------------+-----------------------)--------    (A)
+    //          m-                      v                       m+
+    //
+    //  --------------------(-----------+-----------------------)--------    (B)
+    //                      m-          v                       m+
+    //
+    // First scale v (and m- and m+) such that the exponent is in the range
+    // [alpha, gamma].
+
+    const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e);
+
+    const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k
+
+    // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma]
+    const diyfp w       = diyfp::mul(v,       c_minus_k);
+    const diyfp w_minus = diyfp::mul(m_minus, c_minus_k);
+    const diyfp w_plus  = diyfp::mul(m_plus,  c_minus_k);
+
+    //  ----(---+---)---------------(---+---)---------------(---+---)----
+    //          w-                      w                       w+
+    //          = c*m-                  = c*v                   = c*m+
+    //
+    // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and
+    // w+ are now off by a small amount.
+    // In fact:
+    //
+    //      w - v * 10^k < 1 ulp
+    //
+    // To account for this inaccuracy, add resp. subtract 1 ulp.
+    //
+    //  --------+---[---------------(---+---)---------------]---+--------
+    //          w-  M-                  w                   M+  w+
+    //
+    // Now any number in [M-, M+] (bounds included) will round to w when input,
+    // regardless of how the input rounding algorithm breaks ties.
+    //
+    // And digit_gen generates the shortest possible such number in [M-, M+].
+    // Note that this does not mean that Grisu2 always generates the shortest
+    // possible number in the interval (m-, m+).
+    const diyfp M_minus(w_minus.f + 1, w_minus.e);
+    const diyfp M_plus (w_plus.f  - 1, w_plus.e );
+
+    decimal_exponent = -cached.k; // = -(-k) = k
+
+    grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus);
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+template <typename FloatType>
+void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)
+{
+    static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3,
+                  "internal error: not enough precision");
+
+    assert(std::isfinite(value));
+    assert(value > 0);
+
+    // If the neighbors (and boundaries) of 'value' are always computed for double-precision
+    // numbers, all float's can be recovered using strtod (and strtof). However, the resulting
+    // decimal representations are not exactly "short".
+    //
+    // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars)
+    // says "value is converted to a string as if by std::sprintf in the default ("C") locale"
+    // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars'
+    // does.
+    // On the other hand, the documentation for 'std::to_chars' requires that "parsing the
+    // representation using the corresponding std::from_chars function recovers value exactly". That
+    // indicates that single precision floating-point numbers should be recovered using
+    // 'std::strtof'.
+    //
+    // NB: If the neighbors are computed for single-precision numbers, there is a single float
+    //     (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision
+    //     value is off by 1 ulp.
+#if 0
+    const boundaries w = compute_boundaries(static_cast<double>(value));
+#else
+    const boundaries w = compute_boundaries(value);
+#endif
+
+    grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);
+}
+
+/*!
+@brief appends a decimal representation of e to buf
+@return a pointer to the element following the exponent.
+@pre -1000 < e < 1000
+*/
+inline char* append_exponent(char* buf, int e)
+{
+    assert(e > -1000);
+    assert(e <  1000);
+
+    if (e < 0)
+    {
+        e = -e;
+        *buf++ = '-';
+    }
+    else
+    {
+        *buf++ = '+';
+    }
+
+    auto k = static_cast<uint32_t>(e);
+    if (k < 10)
+    {
+        // Always print at least two digits in the exponent.
+        // This is for compatibility with printf("%g").
+        *buf++ = '0';
+        *buf++ = static_cast<char>('0' + k);
+    }
+    else if (k < 100)
+    {
+        *buf++ = static_cast<char>('0' + k / 10);
+        k %= 10;
+        *buf++ = static_cast<char>('0' + k);
+    }
+    else
+    {
+        *buf++ = static_cast<char>('0' + k / 100);
+        k %= 100;
+        *buf++ = static_cast<char>('0' + k / 10);
+        k %= 10;
+        *buf++ = static_cast<char>('0' + k);
+    }
+
+    return buf;
+}
+
+/*!
+@brief prettify v = buf * 10^decimal_exponent
+
+If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point
+notation. Otherwise it will be printed in exponential notation.
+
+@pre min_exp < 0
+@pre max_exp > 0
+*/
+inline char* format_buffer(char* buf, int len, int decimal_exponent,
+                           int min_exp, int max_exp)
+{
+    assert(min_exp < 0);
+    assert(max_exp > 0);
+
+    const int k = len;
+    const int n = len + decimal_exponent;
+
+    // v = buf * 10^(n-k)
+    // k is the length of the buffer (number of decimal digits)
+    // n is the position of the decimal point relative to the start of the buffer.
+
+    if (k <= n and n <= max_exp)
+    {
+        // digits[000]
+        // len <= max_exp + 2
+
+        std::memset(buf + k, '0', static_cast<size_t>(n - k));
+        // Make it look like a floating-point number (#362, #378)
+        buf[n + 0] = '.';
+        buf[n + 1] = '0';
+        return buf + (n + 2);
+    }
+
+    if (0 < n and n <= max_exp)
+    {
+        // dig.its
+        // len <= max_digits10 + 1
+
+        assert(k > n);
+
+        std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n));
+        buf[n] = '.';
+        return buf + (k + 1);
+    }
+
+    if (min_exp < n and n <= 0)
+    {
+        // 0.[000]digits
+        // len <= 2 + (-min_exp - 1) + max_digits10
+
+        std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k));
+        buf[0] = '0';
+        buf[1] = '.';
+        std::memset(buf + 2, '0', static_cast<size_t>(-n));
+        return buf + (2 + (-n) + k);
+    }
+
+    if (k == 1)
+    {
+        // dE+123
+        // len <= 1 + 5
+
+        buf += 1;
+    }
+    else
+    {
+        // d.igitsE+123
+        // len <= max_digits10 + 1 + 5
+
+        std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1));
+        buf[1] = '.';
+        buf += 1 + k;
+    }
+
+    *buf++ = 'e';
+    return append_exponent(buf, n - 1);
+}
+
+} // namespace dtoa_impl
+
+/*!
+@brief generates a decimal representation of the floating-point number value in [first, last).
+
+The format of the resulting decimal representation is similar to printf's %g
+format. Returns an iterator pointing past-the-end of the decimal representation.
+
+@note The input number must be finite, i.e. NaN's and Inf's are not supported.
+@note The buffer must be large enough.
+@note The result is NOT null-terminated.
+*/
+template <typename FloatType>
+char* to_chars(char* first, const char* last, FloatType value)
+{
+    static_cast<void>(last); // maybe unused - fix warning
+    assert(std::isfinite(value));
+
+    // Use signbit(value) instead of (value < 0) since signbit works for -0.
+    if (std::signbit(value))
+    {
+        value = -value;
+        *first++ = '-';
+    }
+
+    if (value == 0) // +-0
+    {
+        *first++ = '0';
+        // Make it look like a floating-point number (#362, #378)
+        *first++ = '.';
+        *first++ = '0';
+        return first;
+    }
+
+    assert(last - first >= std::numeric_limits<FloatType>::max_digits10);
+
+    // Compute v = buffer * 10^decimal_exponent.
+    // The decimal digits are stored in the buffer, which needs to be interpreted
+    // as an unsigned decimal integer.
+    // len is the length of the buffer, i.e. the number of decimal digits.
+    int len = 0;
+    int decimal_exponent = 0;
+    dtoa_impl::grisu2(first, len, decimal_exponent, value);
+
+    assert(len <= std::numeric_limits<FloatType>::max_digits10);
+
+    // Format the buffer like printf("%.*g", prec, value)
+    constexpr int kMinExp = -4;
+    // Use digits10 here to increase compatibility with version 2.
+    constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10;
+
+    assert(last - first >= kMaxExp + 2);
+    assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10);
+    assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6);
+
+    return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);
+}
+
+} // namespace detail
+} // namespace nlohmann
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// serialization //
+///////////////////
+
+/// how to treat decoding errors
+enum class error_handler_t
+{
+    strict,  ///< throw a type_error exception in case of invalid UTF-8
+    replace, ///< replace invalid UTF-8 sequences with U+FFFD
+    ignore   ///< ignore invalid UTF-8 sequences
+};
+
+template<typename BasicJsonType>
+class serializer
+{
+    using string_t = typename BasicJsonType::string_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    static constexpr uint8_t UTF8_ACCEPT = 0;
+    static constexpr uint8_t UTF8_REJECT = 1;
+
+  public:
+    /*!
+    @param[in] s  output stream to serialize to
+    @param[in] ichar  indentation character to use
+    @param[in] error_handler_  how to react on decoding errors
+    */
+    serializer(output_adapter_t<char> s, const char ichar,
+               error_handler_t error_handler_ = error_handler_t::strict)
+        : o(std::move(s))
+        , loc(std::localeconv())
+        , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep))
+        , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point))
+        , indent_char(ichar)
+        , indent_string(512, indent_char)
+        , error_handler(error_handler_)
+    {}
+
+    // delete because of pointer members
+    serializer(const serializer&) = delete;
+    serializer& operator=(const serializer&) = delete;
+    serializer(serializer&&) = delete;
+    serializer& operator=(serializer&&) = delete;
+    ~serializer() = default;
+
+    /*!
+    @brief internal implementation of the serialization function
+
+    This function is called by the public member function dump and organizes
+    the serialization internally. The indentation level is propagated as
+    additional parameter. In case of arrays and objects, the function is
+    called recursively.
+
+    - strings and object keys are escaped using `escape_string()`
+    - integer numbers are converted implicitly via `operator<<`
+    - floating-point numbers are converted to a string using `"%g"` format
+
+    @param[in] val             value to serialize
+    @param[in] pretty_print    whether the output shall be pretty-printed
+    @param[in] indent_step     the indent level
+    @param[in] current_indent  the current indent level (only used internally)
+    */
+    void dump(const BasicJsonType& val, const bool pretty_print,
+              const bool ensure_ascii,
+              const unsigned int indent_step,
+              const unsigned int current_indent = 0)
+    {
+        switch (val.m_type)
+        {
+            case value_t::object:
+            {
+                if (val.m_value.object->empty())
+                {
+                    o->write_characters("{}", 2);
+                    return;
+                }
+
+                if (pretty_print)
+                {
+                    o->write_characters("{\n", 2);
+
+                    // variable to hold indentation for recursive calls
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
+
+                    // first n-1 elements
+                    auto i = val.m_value.object->cbegin();
+                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                    {
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        o->write_character('\"');
+                        dump_escaped(i->first, ensure_ascii);
+                        o->write_characters("\": ", 3);
+                        dump(i->second, true, ensure_ascii, indent_step, new_indent);
+                        o->write_characters(",\n", 2);
+                    }
+
+                    // last element
+                    assert(i != val.m_value.object->cend());
+                    assert(std::next(i) == val.m_value.object->cend());
+                    o->write_characters(indent_string.c_str(), new_indent);
+                    o->write_character('\"');
+                    dump_escaped(i->first, ensure_ascii);
+                    o->write_characters("\": ", 3);
+                    dump(i->second, true, ensure_ascii, indent_step, new_indent);
+
+                    o->write_character('\n');
+                    o->write_characters(indent_string.c_str(), current_indent);
+                    o->write_character('}');
+                }
+                else
+                {
+                    o->write_character('{');
+
+                    // first n-1 elements
+                    auto i = val.m_value.object->cbegin();
+                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                    {
+                        o->write_character('\"');
+                        dump_escaped(i->first, ensure_ascii);
+                        o->write_characters("\":", 2);
+                        dump(i->second, false, ensure_ascii, indent_step, current_indent);
+                        o->write_character(',');
+                    }
+
+                    // last element
+                    assert(i != val.m_value.object->cend());
+                    assert(std::next(i) == val.m_value.object->cend());
+                    o->write_character('\"');
+                    dump_escaped(i->first, ensure_ascii);
+                    o->write_characters("\":", 2);
+                    dump(i->second, false, ensure_ascii, indent_step, current_indent);
+
+                    o->write_character('}');
+                }
+
+                return;
+            }
+
+            case value_t::array:
+            {
+                if (val.m_value.array->empty())
+                {
+                    o->write_characters("[]", 2);
+                    return;
+                }
+
+                if (pretty_print)
+                {
+                    o->write_characters("[\n", 2);
+
+                    // variable to hold indentation for recursive calls
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
+
+                    // first n-1 elements
+                    for (auto i = val.m_value.array->cbegin();
+                            i != val.m_value.array->cend() - 1; ++i)
+                    {
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        dump(*i, true, ensure_ascii, indent_step, new_indent);
+                        o->write_characters(",\n", 2);
+                    }
+
+                    // last element
+                    assert(not val.m_value.array->empty());
+                    o->write_characters(indent_string.c_str(), new_indent);
+                    dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);
+
+                    o->write_character('\n');
+                    o->write_characters(indent_string.c_str(), current_indent);
+                    o->write_character(']');
+                }
+                else
+                {
+                    o->write_character('[');
+
+                    // first n-1 elements
+                    for (auto i = val.m_value.array->cbegin();
+                            i != val.m_value.array->cend() - 1; ++i)
+                    {
+                        dump(*i, false, ensure_ascii, indent_step, current_indent);
+                        o->write_character(',');
+                    }
+
+                    // last element
+                    assert(not val.m_value.array->empty());
+                    dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);
+
+                    o->write_character(']');
+                }
+
+                return;
+            }
+
+            case value_t::string:
+            {
+                o->write_character('\"');
+                dump_escaped(*val.m_value.string, ensure_ascii);
+                o->write_character('\"');
+                return;
+            }
+
+            case value_t::boolean:
+            {
+                if (val.m_value.boolean)
+                {
+                    o->write_characters("true", 4);
+                }
+                else
+                {
+                    o->write_characters("false", 5);
+                }
+                return;
+            }
+
+            case value_t::number_integer:
+            {
+                dump_integer(val.m_value.number_integer);
+                return;
+            }
+
+            case value_t::number_unsigned:
+            {
+                dump_integer(val.m_value.number_unsigned);
+                return;
+            }
+
+            case value_t::number_float:
+            {
+                dump_float(val.m_value.number_float);
+                return;
+            }
+
+            case value_t::discarded:
+            {
+                o->write_characters("<discarded>", 11);
+                return;
+            }
+
+            case value_t::null:
+            {
+                o->write_characters("null", 4);
+                return;
+            }
+        }
+    }
+
+  private:
+    /*!
+    @brief dump escaped string
+
+    Escape a string by replacing certain special characters by a sequence of an
+    escape character (backslash) and another character and other control
+    characters by a sequence of "\u" followed by a four-digit hex
+    representation. The escaped string is written to output stream @a o.
+
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+
+    @complexity Linear in the length of string @a s.
+    */
+    void dump_escaped(const string_t& s, const bool ensure_ascii)
+    {
+        uint32_t codepoint;
+        uint8_t state = UTF8_ACCEPT;
+        std::size_t bytes = 0;  // number of bytes written to string_buffer
+
+        // number of bytes written at the point of the last valid byte
+        std::size_t bytes_after_last_accept = 0;
+        std::size_t undumped_chars = 0;
+
+        for (std::size_t i = 0; i < s.size(); ++i)
+        {
+            const auto byte = static_cast<uint8_t>(s[i]);
+
+            switch (decode(state, codepoint, byte))
+            {
+                case UTF8_ACCEPT:  // decode found a new code point
+                {
+                    switch (codepoint)
+                    {
+                        case 0x08: // backspace
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'b';
+                            break;
+                        }
+
+                        case 0x09: // horizontal tab
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 't';
+                            break;
+                        }
+
+                        case 0x0A: // newline
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'n';
+                            break;
+                        }
+
+                        case 0x0C: // formfeed
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'f';
+                            break;
+                        }
+
+                        case 0x0D: // carriage return
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'r';
+                            break;
+                        }
+
+                        case 0x22: // quotation mark
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = '\"';
+                            break;
+                        }
+
+                        case 0x5C: // reverse solidus
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = '\\';
+                            break;
+                        }
+
+                        default:
+                        {
+                            // escape control characters (0x00..0x1F) or, if
+                            // ensure_ascii parameter is used, non-ASCII characters
+                            if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F)))
+                            {
+                                if (codepoint <= 0xFFFF)
+                                {
+                                    (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
+                                                    static_cast<uint16_t>(codepoint));
+                                    bytes += 6;
+                                }
+                                else
+                                {
+                                    (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+                                                    static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)),
+                                                    static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF)));
+                                    bytes += 12;
+                                }
+                            }
+                            else
+                            {
+                                // copy byte to buffer (all previous bytes
+                                // been copied have in default case above)
+                                string_buffer[bytes++] = s[i];
+                            }
+                            break;
+                        }
+                    }
+
+                    // write buffer and reset index; there must be 13 bytes
+                    // left, as this is the maximal number of bytes to be
+                    // written ("\uxxxx\uxxxx\0") for one code point
+                    if (string_buffer.size() - bytes < 13)
+                    {
+                        o->write_characters(string_buffer.data(), bytes);
+                        bytes = 0;
+                    }
+
+                    // remember the byte position of this accept
+                    bytes_after_last_accept = bytes;
+                    undumped_chars = 0;
+                    break;
+                }
+
+                case UTF8_REJECT:  // decode found invalid UTF-8 byte
+                {
+                    switch (error_handler)
+                    {
+                        case error_handler_t::strict:
+                        {
+                            std::string sn(3, '\0');
+                            (std::snprintf)(&sn[0], sn.size(), "%.2X", byte);
+                            JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
+                        }
+
+                        case error_handler_t::ignore:
+                        case error_handler_t::replace:
+                        {
+                            // in case we saw this character the first time, we
+                            // would like to read it again, because the byte
+                            // may be OK for itself, but just not OK for the
+                            // previous sequence
+                            if (undumped_chars > 0)
+                            {
+                                --i;
+                            }
+
+                            // reset length buffer to the last accepted index;
+                            // thus removing/ignoring the invalid characters
+                            bytes = bytes_after_last_accept;
+
+                            if (error_handler == error_handler_t::replace)
+                            {
+                                // add a replacement character
+                                if (ensure_ascii)
+                                {
+                                    string_buffer[bytes++] = '\\';
+                                    string_buffer[bytes++] = 'u';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'd';
+                                }
+                                else
+                                {
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF');
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
+                                }
+                                bytes_after_last_accept = bytes;
+                            }
+
+                            undumped_chars = 0;
+
+                            // continue processing the string
+                            state = UTF8_ACCEPT;
+                            break;
+                        }
+                    }
+                    break;
+                }
+
+                default:  // decode found yet incomplete multi-byte code point
+                {
+                    if (not ensure_ascii)
+                    {
+                        // code point will not be escaped - copy byte to buffer
+                        string_buffer[bytes++] = s[i];
+                    }
+                    ++undumped_chars;
+                    break;
+                }
+            }
+        }
+
+        // we finished processing the string
+        if (JSON_LIKELY(state == UTF8_ACCEPT))
+        {
+            // write buffer
+            if (bytes > 0)
+            {
+                o->write_characters(string_buffer.data(), bytes);
+            }
+        }
+        else
+        {
+            // we finish reading, but do not accept: string was incomplete
+            switch (error_handler)
+            {
+                case error_handler_t::strict:
+                {
+                    std::string sn(3, '\0');
+                    (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
+                    JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
+                }
+
+                case error_handler_t::ignore:
+                {
+                    // write all accepted bytes
+                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
+                    break;
+                }
+
+                case error_handler_t::replace:
+                {
+                    // write all accepted bytes
+                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
+                    // add a replacement character
+                    if (ensure_ascii)
+                    {
+                        o->write_characters("\\ufffd", 6);
+                    }
+                    else
+                    {
+                        o->write_characters("\xEF\xBF\xBD", 3);
+                    }
+                    break;
+                }
+            }
+        }
+    }
+
+    /*!
+    @brief dump an integer
+
+    Dump a given integer to output stream @a o. Works internally with
+    @a number_buffer.
+
+    @param[in] x  integer number (signed or unsigned) to dump
+    @tparam NumberType either @a number_integer_t or @a number_unsigned_t
+    */
+    template<typename NumberType, detail::enable_if_t<
+                 std::is_same<NumberType, number_unsigned_t>::value or
+                 std::is_same<NumberType, number_integer_t>::value,
+                 int> = 0>
+    void dump_integer(NumberType x)
+    {
+        // special case for "0"
+        if (x == 0)
+        {
+            o->write_character('0');
+            return;
+        }
+
+        const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0);  // see issue #755
+        std::size_t i = 0;
+
+        while (x != 0)
+        {
+            // spare 1 byte for '\0'
+            assert(i < number_buffer.size() - 1);
+
+            const auto digit = std::labs(static_cast<long>(x % 10));
+            number_buffer[i++] = static_cast<char>('0' + digit);
+            x /= 10;
+        }
+
+        if (is_negative)
+        {
+            // make sure there is capacity for the '-'
+            assert(i < number_buffer.size() - 2);
+            number_buffer[i++] = '-';
+        }
+
+        std::reverse(number_buffer.begin(), number_buffer.begin() + i);
+        o->write_characters(number_buffer.data(), i);
+    }
+
+    /*!
+    @brief dump a floating-point number
+
+    Dump a given floating-point number to output stream @a o. Works internally
+    with @a number_buffer.
+
+    @param[in] x  floating-point number to dump
+    */
+    void dump_float(number_float_t x)
+    {
+        // NaN / inf
+        if (not std::isfinite(x))
+        {
+            o->write_characters("null", 4);
+            return;
+        }
+
+        // If number_float_t is an IEEE-754 single or double precision number,
+        // use the Grisu2 algorithm to produce short numbers which are
+        // guaranteed to round-trip, using strtof and strtod, resp.
+        //
+        // NB: The test below works if <long double> == <double>.
+        static constexpr bool is_ieee_single_or_double
+            = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or
+              (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024);
+
+        dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>());
+    }
+
+    void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/)
+    {
+        char* begin = number_buffer.data();
+        char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x);
+
+        o->write_characters(begin, static_cast<size_t>(end - begin));
+    }
+
+    void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/)
+    {
+        // get number of digits for a float -> text -> float round-trip
+        static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;
+
+        // the actual conversion
+        std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
+
+        // negative value indicates an error
+        assert(len > 0);
+        // check if buffer was large enough
+        assert(static_cast<std::size_t>(len) < number_buffer.size());
+
+        // erase thousands separator
+        if (thousands_sep != '\0')
+        {
+            const auto end = std::remove(number_buffer.begin(),
+                                         number_buffer.begin() + len, thousands_sep);
+            std::fill(end, number_buffer.end(), '\0');
+            assert((end - number_buffer.begin()) <= len);
+            len = (end - number_buffer.begin());
+        }
+
+        // convert decimal point to '.'
+        if (decimal_point != '\0' and decimal_point != '.')
+        {
+            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            if (dec_pos != number_buffer.end())
+            {
+                *dec_pos = '.';
+            }
+        }
+
+        o->write_characters(number_buffer.data(), static_cast<std::size_t>(len));
+
+        // determine if need to append ".0"
+        const bool value_is_int_like =
+            std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1,
+                         [](char c)
+        {
+            return (c == '.' or c == 'e');
+        });
+
+        if (value_is_int_like)
+        {
+            o->write_characters(".0", 2);
+        }
+    }
+
+    /*!
+    @brief check whether a string is UTF-8 encoded
+
+    The function checks each byte of a string whether it is UTF-8 encoded. The
+    result of the check is stored in the @a state parameter. The function must
+    be called initially with state 0 (accept). State 1 means the string must
+    be rejected, because the current byte is not allowed. If the string is
+    completely processed, but the state is non-zero, the string ended
+    prematurely; that is, the last byte indicated more bytes should have
+    followed.
+
+    @param[in,out] state  the state of the decoding
+    @param[in,out] codep  codepoint (valid only if resulting state is UTF8_ACCEPT)
+    @param[in] byte       next byte to decode
+    @return               new state
+
+    @note The function has been edited: a std::array is used.
+
+    @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+    @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+    */
+    static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept
+    {
+        static const std::array<uint8_t, 400> utf8d =
+        {
+            {
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F
+                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F
+                7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF
+                8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF
+                0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF
+                0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF
+                0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
+                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
+                1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
+                1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
+                1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8
+            }
+        };
+
+        const uint8_t type = utf8d[byte];
+
+        codep = (state != UTF8_ACCEPT)
+                ? (byte & 0x3fu) | (codep << 6)
+                : static_cast<uint32_t>(0xff >> type) & (byte);
+
+        state = utf8d[256u + state * 16u + type];
+        return state;
+    }
+
+  private:
+    /// the output of the serializer
+    output_adapter_t<char> o = nullptr;
+
+    /// a (hopefully) large enough character buffer
+    std::array<char, 64> number_buffer{{}};
+
+    /// the locale
+    const std::lconv* loc = nullptr;
+    /// the locale's thousand separator character
+    const char thousands_sep = '\0';
+    /// the locale's decimal point character
+    const char decimal_point = '\0';
+
+    /// string buffer
+    std::array<char, 512> string_buffer{{}};
+
+    /// the indentation character
+    const char indent_char;
+    /// the indentation string
+    string_t indent_string;
+
+    /// error_handler how to react on decoding errors
+    const error_handler_t error_handler;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/json_ref.hpp>
+
+
+#include <initializer_list>
+#include <utility>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template<typename BasicJsonType>
+class json_ref
+{
+  public:
+    using value_type = BasicJsonType;
+
+    json_ref(value_type&& value)
+        : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true)
+    {}
+
+    json_ref(const value_type& value)
+        : value_ref(const_cast<value_type*>(&value)), is_rvalue(false)
+    {}
+
+    json_ref(std::initializer_list<json_ref> init)
+        : owned_value(init), value_ref(&owned_value), is_rvalue(true)
+    {}
+
+    template <
+        class... Args,
+        enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 >
+    json_ref(Args && ... args)
+        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value),
+          is_rvalue(true) {}
+
+    // class should be movable only
+    json_ref(json_ref&&) = default;
+    json_ref(const json_ref&) = delete;
+    json_ref& operator=(const json_ref&) = delete;
+    json_ref& operator=(json_ref&&) = delete;
+    ~json_ref() = default;
+
+    value_type moved_or_copied() const
+    {
+        if (is_rvalue)
+        {
+            return std::move(*value_ref);
+        }
+        return *value_ref;
+    }
+
+    value_type const& operator*() const
+    {
+        return *static_cast<value_type const*>(value_ref);
+    }
+
+    value_type const* operator->() const
+    {
+        return static_cast<value_type const*>(value_ref);
+    }
+
+  private:
+    mutable value_type owned_value = nullptr;
+    value_type* value_ref = nullptr;
+    const bool is_rvalue;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/json_pointer.hpp>
+
+
+#include <cassert> // assert
+#include <numeric> // accumulate
+#include <string> // string
+#include <vector> // vector
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+template<typename BasicJsonType>
+class json_pointer
+{
+    // allow basic_json to access private members
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    friend class basic_json;
+
+  public:
+    /*!
+    @brief create JSON pointer
+
+    Create a JSON pointer according to the syntax described in
+    [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3).
+
+    @param[in] s  string representing the JSON pointer; if omitted, the empty
+                  string is assumed which references the whole JSON value
+
+    @throw parse_error.107 if the given JSON pointer @a s is nonempty and does
+                           not begin with a slash (`/`); see example below
+
+    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is
+    not followed by `0` (representing `~`) or `1` (representing `/`); see
+    example below
+
+    @liveexample{The example shows the construction several valid JSON pointers
+    as well as the exceptional behavior.,json_pointer}
+
+    @since version 2.0.0
+    */
+    explicit json_pointer(const std::string& s = "")
+        : reference_tokens(split(s))
+    {}
+
+    /*!
+    @brief return a string representation of the JSON pointer
+
+    @invariant For each JSON pointer `ptr`, it holds:
+    @code {.cpp}
+    ptr == json_pointer(ptr.to_string());
+    @endcode
+
+    @return a string representation of the JSON pointer
+
+    @liveexample{The example shows the result of `to_string`.,
+    json_pointer__to_string}
+
+    @since version 2.0.0
+    */
+    std::string to_string() const
+    {
+        return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
+                               std::string{},
+                               [](const std::string & a, const std::string & b)
+        {
+            return a + "/" + escape(b);
+        });
+    }
+
+    /// @copydoc to_string()
+    operator std::string() const
+    {
+        return to_string();
+    }
+
+    /*!
+    @param[in] s  reference token to be converted into an array index
+
+    @return integer representation of @a s
+
+    @throw out_of_range.404 if string @a s could not be converted to an integer
+    */
+    static int array_index(const std::string& s)
+    {
+        std::size_t processed_chars = 0;
+        const int res = std::stoi(s, &processed_chars);
+
+        // check if the string was completely read
+        if (JSON_UNLIKELY(processed_chars != s.size()))
+        {
+            JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
+        }
+
+        return res;
+    }
+
+  private:
+    /*!
+    @brief remove and return last reference pointer
+    @throw out_of_range.405 if JSON pointer has no parent
+    */
+    std::string pop_back()
+    {
+        if (JSON_UNLIKELY(is_root()))
+        {
+            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
+        }
+
+        auto last = reference_tokens.back();
+        reference_tokens.pop_back();
+        return last;
+    }
+
+    /// return whether pointer points to the root document
+    bool is_root() const noexcept
+    {
+        return reference_tokens.empty();
+    }
+
+    json_pointer top() const
+    {
+        if (JSON_UNLIKELY(is_root()))
+        {
+            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
+        }
+
+        json_pointer result = *this;
+        result.reference_tokens = {reference_tokens[0]};
+        return result;
+    }
+
+    /*!
+    @brief create and return a reference to the pointed to value
+
+    @complexity Linear in the number of reference tokens.
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.313 if value cannot be unflattened
+    */
+    BasicJsonType& get_and_create(BasicJsonType& j) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        auto result = &j;
+
+        // in case no reference tokens exist, return a reference to the JSON value
+        // j which will be overwritten by a primitive value
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (result->m_type)
+            {
+                case detail::value_t::null:
+                {
+                    if (reference_token == "0")
+                    {
+                        // start a new array if reference token is 0
+                        result = &result->operator[](0);
+                    }
+                    else
+                    {
+                        // start a new object otherwise
+                        result = &result->operator[](reference_token);
+                    }
+                    break;
+                }
+
+                case detail::value_t::object:
+                {
+                    // create an entry in the object
+                    result = &result->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    // create an entry in the array
+                    JSON_TRY
+                    {
+                        result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                /*
+                The following code is only reached if there exists a reference
+                token _and_ the current value is primitive. In this case, we have
+                an error situation, because primitive values may only occur as
+                single value; that is, with an empty list of reference tokens.
+                */
+                default:
+                    JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
+            }
+        }
+
+        return *result;
+    }
+
+    /*!
+    @brief return a reference to the pointed to value
+
+    @note This version does not throw if a value is not present, but tries to
+          create nested values instead. For instance, calling this function
+          with pointer `"/this/that"` on a null value is equivalent to calling
+          `operator[]("this").operator[]("that")` on that value, effectively
+          changing the null value to an object.
+
+    @param[in] ptr  a JSON value
+
+    @return reference to the JSON value pointed to by the JSON pointer
+
+    @complexity Linear in the length of the JSON pointer.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    BasicJsonType& get_unchecked(BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            // convert null values to arrays or objects before continuing
+            if (ptr->m_type == detail::value_t::null)
+            {
+                // check if reference token is a number
+                const bool nums =
+                    std::all_of(reference_token.begin(), reference_token.end(),
+                                [](const char x)
+                {
+                    return (x >= '0' and x <= '9');
+                });
+
+                // change value to array for numbers or "-" or to object otherwise
+                *ptr = (nums or reference_token == "-")
+                       ? detail::value_t::array
+                       : detail::value_t::object;
+            }
+
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // use unchecked object access
+                    ptr = &ptr->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    if (reference_token == "-")
+                    {
+                        // explicitly treat "-" as index beyond the end
+                        ptr = &ptr->operator[](ptr->m_value.array->size());
+                    }
+                    else
+                    {
+                        // convert array index to number; unchecked access
+                        JSON_TRY
+                        {
+                            ptr = &ptr->operator[](
+                                static_cast<size_type>(array_index(reference_token)));
+                        }
+                        JSON_CATCH(std::invalid_argument&)
+                        {
+                            JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
+
+    /*!
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    BasicJsonType& get_checked(BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // note: at performs range check
+                    ptr = &ptr->at(reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" always fails the range check
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // note: at performs range check
+                    JSON_TRY
+                    {
+                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
+
+    /*!
+    @brief return a const reference to the pointed to value
+
+    @param[in] ptr  a JSON value
+
+    @return const reference to the JSON value pointed to by the JSON
+    pointer
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // use unchecked object access
+                    ptr = &ptr->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" cannot be used for const access
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // use unchecked array access
+                    JSON_TRY
+                    {
+                        ptr = &ptr->operator[](
+                            static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
+
+    /*!
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    const BasicJsonType& get_checked(const BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // note: at performs range check
+                    ptr = &ptr->at(reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" always fails the range check
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // note: at performs range check
+                    JSON_TRY
+                    {
+                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
+
+    /*!
+    @brief split the string input to reference tokens
+
+    @note This function is only called by the json_pointer constructor.
+          All exceptions below are documented there.
+
+    @throw parse_error.107  if the pointer is not empty or begins with '/'
+    @throw parse_error.108  if character '~' is not followed by '0' or '1'
+    */
+    static std::vector<std::string> split(const std::string& reference_string)
+    {
+        std::vector<std::string> result;
+
+        // special case: empty reference string -> no reference tokens
+        if (reference_string.empty())
+        {
+            return result;
+        }
+
+        // check if nonempty reference string begins with slash
+        if (JSON_UNLIKELY(reference_string[0] != '/'))
+        {
+            JSON_THROW(detail::parse_error::create(107, 1,
+                                                   "JSON pointer must be empty or begin with '/' - was: '" +
+                                                   reference_string + "'"));
+        }
+
+        // extract the reference tokens:
+        // - slash: position of the last read slash (or end of string)
+        // - start: position after the previous slash
+        for (
+            // search for the first slash after the first character
+            std::size_t slash = reference_string.find_first_of('/', 1),
+            // set the beginning of the first reference token
+            start = 1;
+            // we can stop if start == 0 (if slash == std::string::npos)
+            start != 0;
+            // set the beginning of the next reference token
+            // (will eventually be 0 if slash == std::string::npos)
+            start = (slash == std::string::npos) ? 0 : slash + 1,
+            // find next slash
+            slash = reference_string.find_first_of('/', start))
+        {
+            // use the text between the beginning of the reference token
+            // (start) and the last slash (slash).
+            auto reference_token = reference_string.substr(start, slash - start);
+
+            // check reference tokens are properly escaped
+            for (std::size_t pos = reference_token.find_first_of('~');
+                    pos != std::string::npos;
+                    pos = reference_token.find_first_of('~', pos + 1))
+            {
+                assert(reference_token[pos] == '~');
+
+                // ~ must be followed by 0 or 1
+                if (JSON_UNLIKELY(pos == reference_token.size() - 1 or
+                                  (reference_token[pos + 1] != '0' and
+                                   reference_token[pos + 1] != '1')))
+                {
+                    JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'"));
+                }
+            }
+
+            // finally, store the reference token
+            unescape(reference_token);
+            result.push_back(reference_token);
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief replace all occurrences of a substring by another string
+
+    @param[in,out] s  the string to manipulate; changed so that all
+                   occurrences of @a f are replaced with @a t
+    @param[in]     f  the substring to replace with @a t
+    @param[in]     t  the string to replace @a f
+
+    @pre The search string @a f must not be empty. **This precondition is
+    enforced with an assertion.**
+
+    @since version 2.0.0
+    */
+    static void replace_substring(std::string& s, const std::string& f,
+                                  const std::string& t)
+    {
+        assert(not f.empty());
+        for (auto pos = s.find(f);                // find first occurrence of f
+                pos != std::string::npos;         // make sure f was found
+                s.replace(pos, f.size(), t),      // replace with t, and
+                pos = s.find(f, pos + t.size()))  // find next occurrence of f
+        {}
+    }
+
+    /// escape "~" to "~0" and "/" to "~1"
+    static std::string escape(std::string s)
+    {
+        replace_substring(s, "~", "~0");
+        replace_substring(s, "/", "~1");
+        return s;
+    }
+
+    /// unescape "~1" to tilde and "~0" to slash (order is important!)
+    static void unescape(std::string& s)
+    {
+        replace_substring(s, "~1", "/");
+        replace_substring(s, "~0", "~");
+    }
+
+    /*!
+    @param[in] reference_string  the reference string to the current value
+    @param[in] value             the value to consider
+    @param[in,out] result        the result object to insert values to
+
+    @note Empty objects or arrays are flattened to `null`.
+    */
+    static void flatten(const std::string& reference_string,
+                        const BasicJsonType& value,
+                        BasicJsonType& result)
+    {
+        switch (value.m_type)
+        {
+            case detail::value_t::array:
+            {
+                if (value.m_value.array->empty())
+                {
+                    // flatten empty array as null
+                    result[reference_string] = nullptr;
+                }
+                else
+                {
+                    // iterate array and use index as reference string
+                    for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
+                    {
+                        flatten(reference_string + "/" + std::to_string(i),
+                                value.m_value.array->operator[](i), result);
+                    }
+                }
+                break;
+            }
+
+            case detail::value_t::object:
+            {
+                if (value.m_value.object->empty())
+                {
+                    // flatten empty object as null
+                    result[reference_string] = nullptr;
+                }
+                else
+                {
+                    // iterate object and use keys as reference string
+                    for (const auto& element : *value.m_value.object)
+                    {
+                        flatten(reference_string + "/" + escape(element.first), element.second, result);
+                    }
+                }
+                break;
+            }
+
+            default:
+            {
+                // add primitive value with its reference string
+                result[reference_string] = value;
+                break;
+            }
+        }
+    }
+
+    /*!
+    @param[in] value  flattened JSON
+
+    @return unflattened JSON
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitive
+    @throw type_error.313  if value cannot be unflattened
+    */
+    static BasicJsonType
+    unflatten(const BasicJsonType& value)
+    {
+        if (JSON_UNLIKELY(not value.is_object()))
+        {
+            JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
+        }
+
+        BasicJsonType result;
+
+        // iterate the JSON object values
+        for (const auto& element : *value.m_value.object)
+        {
+            if (JSON_UNLIKELY(not element.second.is_primitive()))
+            {
+                JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+            }
+
+            // assign value to reference pointed to by JSON pointer; Note that if
+            // the JSON pointer is "" (i.e., points to the whole value), function
+            // get_and_create returns a reference to result itself. An assignment
+            // will then create a primitive value.
+            json_pointer(element.first).get_and_create(result) = element.second;
+        }
+
+        return result;
+    }
+
+    friend bool operator==(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept
+    {
+        return (lhs.reference_tokens == rhs.reference_tokens);
+    }
+
+    friend bool operator!=(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept
+    {
+        return not (lhs == rhs);
+    }
+
+    /// the reference tokens
+    std::vector<std::string> reference_tokens;
+};
+}  // namespace nlohmann
+
+// #include <nlohmann/adl_serializer.hpp>
+
+
+#include <utility>
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
+
+// #include <nlohmann/detail/conversions/to_json.hpp>
+
+
+namespace nlohmann
+{
+
+template<typename, typename>
+struct adl_serializer
+{
+    /*!
+    @brief convert a JSON value to any value type
+
+    This function is usually called by the `get()` function of the
+    @ref basic_json class (either explicit or via conversion operators).
+
+    @param[in] j        JSON value to read from
+    @param[in,out] val  value to write to
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static auto from_json(BasicJsonType&& j, ValueType& val) noexcept(
+        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
+    -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void())
+    {
+        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
+    }
+
+    /*!
+    @brief convert any value type to a JSON value
+
+    This function is usually called by the constructors of the @ref basic_json
+    class.
+
+    @param[in,out] j  JSON value to write to
+    @param[in] val    value to read from
+    */
+    template <typename BasicJsonType, typename ValueType>
+    static auto to_json(BasicJsonType& j, ValueType&& val) noexcept(
+        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
+    -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void())
+    {
+        ::nlohmann::to_json(j, std::forward<ValueType>(val));
+    }
+};
+
+}  // namespace nlohmann
+
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
+
+/*!
+@brief a class to store JSON values
+
+@tparam ObjectType type for JSON objects (`std::map` by default; will be used
+in @ref object_t)
+@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used
+in @ref array_t)
+@tparam StringType type for JSON strings and object keys (`std::string` by
+default; will be used in @ref string_t)
+@tparam BooleanType type for JSON booleans (`bool` by default; will be used
+in @ref boolean_t)
+@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by
+default; will be used in @ref number_integer_t)
+@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c
+`uint64_t` by default; will be used in @ref number_unsigned_t)
+@tparam NumberFloatType type for JSON floating-point numbers (`double` by
+default; will be used in @ref number_float_t)
+@tparam AllocatorType type of the allocator to use (`std::allocator` by
+default)
+@tparam JSONSerializer the serializer to resolve internal calls to `to_json()`
+and `from_json()` (@ref adl_serializer by default)
+
+@requirement The class satisfies the following concept requirements:
+- Basic
+ - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible):
+   JSON values can be default constructed. The result will be a JSON null
+   value.
+ - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible):
+   A JSON value can be constructed from an rvalue argument.
+ - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible):
+   A JSON value can be copy-constructed from an lvalue expression.
+ - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable):
+   A JSON value van be assigned from an rvalue argument.
+ - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable):
+   A JSON value can be copy-assigned from an lvalue expression.
+ - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible):
+   JSON values can be destructed.
+- Layout
+ - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType):
+   JSON values have
+   [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
+   All non-static data members are private and standard layout types, the
+   class has no virtual functions or (virtual) base classes.
+- Library-wide
+ - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable):
+   JSON values can be compared with `==`, see @ref
+   operator==(const_reference,const_reference).
+ - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable):
+   JSON values can be compared with `<`, see @ref
+   operator<(const_reference,const_reference).
+ - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable):
+   Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of
+   other compatible types, using unqualified function call @ref swap().
+ - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer):
+   JSON values can be compared against `std::nullptr_t` objects which are used
+   to model the `null` value.
+- Container
+ - [Container](https://en.cppreference.com/w/cpp/named_req/Container):
+   JSON values can be used like STL containers and provide iterator access.
+ - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer);
+   JSON values can be used like STL containers and provide reverse iterator
+   access.
+
+@invariant The member variables @a m_value and @a m_type have the following
+relationship:
+- If `m_type == value_t::object`, then `m_value.object != nullptr`.
+- If `m_type == value_t::array`, then `m_value.array != nullptr`.
+- If `m_type == value_t::string`, then `m_value.string != nullptr`.
+The invariants are checked by member function assert_invariant().
+
+@internal
+@note ObjectType trick from http://stackoverflow.com/a/9860911
+@endinternal
+
+@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange
+Format](http://rfc7159.net/rfc7159)
+
+@since version 1.0.0
+
+@nosubgrouping
+*/
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+class basic_json
+{
+  private:
+    template<detail::value_t> friend struct detail::external_constructor;
+    friend ::nlohmann::json_pointer<basic_json>;
+    friend ::nlohmann::detail::parser<basic_json>;
+    friend ::nlohmann::detail::serializer<basic_json>;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::iter_impl;
+    template<typename BasicJsonType, typename CharType>
+    friend class ::nlohmann::detail::binary_writer;
+    template<typename BasicJsonType, typename SAX>
+    friend class ::nlohmann::detail::binary_reader;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::json_sax_dom_parser;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::json_sax_dom_callback_parser;
+
+    /// workaround type for MSVC
+    using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
+
+    // convenience aliases for types residing in namespace detail;
+    using lexer = ::nlohmann::detail::lexer<basic_json>;
+    using parser = ::nlohmann::detail::parser<basic_json>;
+
+    using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t;
+    template<typename BasicJsonType>
+    using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>;
+    template<typename BasicJsonType>
+    using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>;
+    template<typename Iterator>
+    using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>;
+    template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>;
+
+    template<typename CharType>
+    using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>;
+
+    using binary_reader = ::nlohmann::detail::binary_reader<basic_json>;
+    template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>;
+
+    using serializer = ::nlohmann::detail::serializer<basic_json>;
+
+  public:
+    using value_t = detail::value_t;
+    /// JSON Pointer, see @ref nlohmann::json_pointer
+    using json_pointer = ::nlohmann::json_pointer<basic_json>;
+    template<typename T, typename SFINAE>
+    using json_serializer = JSONSerializer<T, SFINAE>;
+    /// how to treat decoding errors
+    using error_handler_t = detail::error_handler_t;
+    /// helper type for initializer lists of basic_json values
+    using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
+
+    using input_format_t = detail::input_format_t;
+    /// SAX interface type, see @ref nlohmann::json_sax
+    using json_sax_t = json_sax<basic_json>;
+
+    ////////////////
+    // exceptions //
+    ////////////////
+
+    /// @name exceptions
+    /// Classes to implement user-defined exceptions.
+    /// @{
+
+    /// @copydoc detail::exception
+    using exception = detail::exception;
+    /// @copydoc detail::parse_error
+    using parse_error = detail::parse_error;
+    /// @copydoc detail::invalid_iterator
+    using invalid_iterator = detail::invalid_iterator;
+    /// @copydoc detail::type_error
+    using type_error = detail::type_error;
+    /// @copydoc detail::out_of_range
+    using out_of_range = detail::out_of_range;
+    /// @copydoc detail::other_error
+    using other_error = detail::other_error;
+
+    /// @}
+
+
+    /////////////////////
+    // container types //
+    /////////////////////
+
+    /// @name container types
+    /// The canonic container types to use @ref basic_json like any other STL
+    /// container.
+    /// @{
+
+    /// the type of elements in a basic_json container
+    using value_type = basic_json;
+
+    /// the type of an element reference
+    using reference = value_type&;
+    /// the type of an element const reference
+    using const_reference = const value_type&;
+
+    /// a type to represent differences between iterators
+    using difference_type = std::ptrdiff_t;
+    /// a type to represent container sizes
+    using size_type = std::size_t;
+
+    /// the allocator type
+    using allocator_type = AllocatorType<basic_json>;
+
+    /// the type of an element pointer
+    using pointer = typename std::allocator_traits<allocator_type>::pointer;
+    /// the type of an element const pointer
+    using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
+
+    /// an iterator for a basic_json container
+    using iterator = iter_impl<basic_json>;
+    /// a const iterator for a basic_json container
+    using const_iterator = iter_impl<const basic_json>;
+    /// a reverse iterator for a basic_json container
+    using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>;
+    /// a const reverse iterator for a basic_json container
+    using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>;
+
+    /// @}
+
+
+    /*!
+    @brief returns the allocator associated with the container
+    */
+    static allocator_type get_allocator()
+    {
+        return allocator_type();
+    }
+
+    /*!
+    @brief returns version information on the library
+
+    This function returns a JSON object with information about the library,
+    including the version number and information on the platform and compiler.
+
+    @return JSON object holding version information
+    key         | description
+    ----------- | ---------------
+    `compiler`  | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version).
+    `copyright` | The copyright line for the library as string.
+    `name`      | The name of the library as string.
+    `platform`  | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`.
+    `url`       | The URL of the project as string.
+    `version`   | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string).
+
+    @liveexample{The following code shows an example output of the `meta()`
+    function.,meta}
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @complexity Constant.
+
+    @since 2.1.0
+    */
+    static basic_json meta()
+    {
+        basic_json result;
+
+        result["copyright"] = "(C) 2013-2017 Niels Lohmann";
+        result["name"] = "JSON for Modern C++";
+        result["url"] = "https://github.com/nlohmann/json";
+        result["version"]["string"] =
+            std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." +
+            std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." +
+            std::to_string(NLOHMANN_JSON_VERSION_PATCH);
+        result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR;
+        result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR;
+        result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH;
+
+#ifdef _WIN32
+        result["platform"] = "win32";
+#elif defined __linux__
+        result["platform"] = "linux";
+#elif defined __APPLE__
+        result["platform"] = "apple";
+#elif defined __unix__
+        result["platform"] = "unix";
+#else
+        result["platform"] = "unknown";
+#endif
+
+#if defined(__ICC) || defined(__INTEL_COMPILER)
+        result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}};
+#elif defined(__clang__)
+        result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
+#elif defined(__GNUC__) || defined(__GNUG__)
+        result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}};
+#elif defined(__HP_cc) || defined(__HP_aCC)
+        result["compiler"] = "hp"
+#elif defined(__IBMCPP__)
+        result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}};
+#elif defined(_MSC_VER)
+        result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}};
+#elif defined(__PGI)
+        result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}};
+#elif defined(__SUNPRO_CC)
+        result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}};
+#else
+        result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}};
+#endif
+
+#ifdef __cplusplus
+        result["compiler"]["c++"] = std::to_string(__cplusplus);
+#else
+        result["compiler"]["c++"] = "unknown";
+#endif
+        return result;
+    }
+
+
+    ///////////////////////////
+    // JSON value data types //
+    ///////////////////////////
+
+    /// @name JSON value data types
+    /// The data types to store a JSON value. These types are derived from
+    /// the template arguments passed to class @ref basic_json.
+    /// @{
+
+#if defined(JSON_HAS_CPP_14)
+    // Use transparent comparator if possible, combined with perfect forwarding
+    // on find() and count() calls prevents unnecessary string construction.
+    using object_comparator_t = std::less<>;
+#else
+    using object_comparator_t = std::less<StringType>;
+#endif
+
+    /*!
+    @brief a type for an object
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows:
+    > An object is an unordered collection of zero or more name/value pairs,
+    > where a name is a string and a value is a string, number, boolean, null,
+    > object, or array.
+
+    To store objects in C++, a type is defined by the template parameters
+    described below.
+
+    @tparam ObjectType  the container to store objects (e.g., `std::map` or
+    `std::unordered_map`)
+    @tparam StringType the type of the keys or names (e.g., `std::string`).
+    The comparison function `std::less<StringType>` is used to order elements
+    inside the container.
+    @tparam AllocatorType the allocator to use for objects (e.g.,
+    `std::allocator`)
+
+    #### Default type
+
+    With the default values for @a ObjectType (`std::map`), @a StringType
+    (`std::string`), and @a AllocatorType (`std::allocator`), the default
+    value for @a object_t is:
+
+    @code {.cpp}
+    std::map<
+      std::string, // key_type
+      basic_json, // value_type
+      std::less<std::string>, // key_compare
+      std::allocator<std::pair<const std::string, basic_json>> // allocator_type
+    >
+    @endcode
+
+    #### Behavior
+
+    The choice of @a object_t influences the behavior of the JSON class. With
+    the default type, objects have the following behavior:
+
+    - When all names are unique, objects will be interoperable in the sense
+      that all software implementations receiving that object will agree on
+      the name-value mappings.
+    - When the names within an object are not unique, it is unspecified which
+      one of the values for a given key will be chosen. For instance,
+      `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or
+      `{"key": 2}`.
+    - Internally, name/value pairs are stored in lexicographical order of the
+      names. Objects will also be serialized (see @ref dump) in this order.
+      For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored
+      and serialized as `{"a": 2, "b": 1}`.
+    - When comparing objects, the order of the name/value pairs is irrelevant.
+      This makes objects interoperable in the sense that they will not be
+      affected by these differences. For instance, `{"b": 1, "a": 2}` and
+      `{"a": 2, "b": 1}` will be treated as equal.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the maximum depth of nesting.
+
+    In this class, the object's limit of nesting is not explicitly constrained.
+    However, a maximum depth of nesting may be introduced by the compiler or
+    runtime environment. A theoretical limit can be queried by calling the
+    @ref max_size function of a JSON object.
+
+    #### Storage
+
+    Objects are stored as pointers in a @ref basic_json type. That is, for any
+    access to object values, a pointer of type `object_t*` must be
+    dereferenced.
+
+    @sa @ref array_t -- type for an array value
+
+    @since version 1.0.0
+
+    @note The order name/value pairs are added to the object is *not*
+    preserved by the library. Therefore, iterating an object may return
+    name/value pairs in a different order than they were originally stored. In
+    fact, keys will be traversed in alphabetical order as `std::map` with
+    `std::less` is used by default. Please note this behavior conforms to [RFC
+    7159](http://rfc7159.net/rfc7159), because any order implements the
+    specified "unordered" nature of JSON objects.
+    */
+    using object_t = ObjectType<StringType,
+          basic_json,
+          object_comparator_t,
+          AllocatorType<std::pair<const StringType,
+          basic_json>>>;
+
+    /*!
+    @brief a type for an array
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows:
+    > An array is an ordered sequence of zero or more values.
+
+    To store objects in C++, a type is defined by the template parameters
+    explained below.
+
+    @tparam ArrayType  container type to store arrays (e.g., `std::vector` or
+    `std::list`)
+    @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`)
+
+    #### Default type
+
+    With the default values for @a ArrayType (`std::vector`) and @a
+    AllocatorType (`std::allocator`), the default value for @a array_t is:
+
+    @code {.cpp}
+    std::vector<
+      basic_json, // value_type
+      std::allocator<basic_json> // allocator_type
+    >
+    @endcode
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the maximum depth of nesting.
+
+    In this class, the array's limit of nesting is not explicitly constrained.
+    However, a maximum depth of nesting may be introduced by the compiler or
+    runtime environment. A theoretical limit can be queried by calling the
+    @ref max_size function of a JSON array.
+
+    #### Storage
+
+    Arrays are stored as pointers in a @ref basic_json type. That is, for any
+    access to array values, a pointer of type `array_t*` must be dereferenced.
+
+    @sa @ref object_t -- type for an object value
+
+    @since version 1.0.0
+    */
+    using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
+
+    /*!
+    @brief a type for a string
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows:
+    > A string is a sequence of zero or more Unicode characters.
+
+    To store objects in C++, a type is defined by the template parameter
+    described below. Unicode values are split by the JSON class into
+    byte-sized characters during deserialization.
+
+    @tparam StringType  the container to store strings (e.g., `std::string`).
+    Note this container is used for keys/names in objects, see @ref object_t.
+
+    #### Default type
+
+    With the default values for @a StringType (`std::string`), the default
+    value for @a string_t is:
+
+    @code {.cpp}
+    std::string
+    @endcode
+
+    #### Encoding
+
+    Strings are stored in UTF-8 encoding. Therefore, functions like
+    `std::string::size()` or `std::string::length()` return the number of
+    bytes in the string rather than the number of characters or glyphs.
+
+    #### String comparison
+
+    [RFC 7159](http://rfc7159.net/rfc7159) states:
+    > Software implementations are typically required to test names of object
+    > members for equality. Implementations that transform the textual
+    > representation into sequences of Unicode code units and then perform the
+    > comparison numerically, code unit by code unit, are interoperable in the
+    > sense that implementations will agree in all cases on equality or
+    > inequality of two strings. For example, implementations that compare
+    > strings with escaped characters unconverted may incorrectly find that
+    > `"a\\b"` and `"a\u005Cb"` are not equal.
+
+    This implementation is interoperable as it does compare strings code unit
+    by code unit.
+
+    #### Storage
+
+    String values are stored as pointers in a @ref basic_json type. That is,
+    for any access to string values, a pointer of type `string_t*` must be
+    dereferenced.
+
+    @since version 1.0.0
+    */
+    using string_t = StringType;
+
+    /*!
+    @brief a type for a boolean
+
+    [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a
+    type which differentiates the two literals `true` and `false`.
+
+    To store objects in C++, a type is defined by the template parameter @a
+    BooleanType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a BooleanType (`bool`), the default value for
+    @a boolean_t is:
+
+    @code {.cpp}
+    bool
+    @endcode
+
+    #### Storage
+
+    Boolean values are stored directly inside a @ref basic_json type.
+
+    @since version 1.0.0
+    */
+    using boolean_t = BooleanType;
+
+    /*!
+    @brief a type for a number (integer)
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
+    > The representation of numbers is similar to that used in most
+    > programming languages. A number is represented in base 10 using decimal
+    > digits. It contains an integer component that may be prefixed with an
+    > optional minus sign, which may be followed by a fraction part and/or an
+    > exponent part. Leading zeros are not allowed. (...) Numeric values that
+    > cannot be represented in the grammar below (such as Infinity and NaN)
+    > are not permitted.
+
+    This description includes both integer and floating-point numbers.
+    However, C++ allows more precise storage if it is known whether the number
+    is a signed integer, an unsigned integer or a floating-point number.
+    Therefore, three different types, @ref number_integer_t, @ref
+    number_unsigned_t and @ref number_float_t are used.
+
+    To store integer numbers in C++, a type is defined by the template
+    parameter @a NumberIntegerType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a NumberIntegerType (`int64_t`), the default
+    value for @a number_integer_t is:
+
+    @code {.cpp}
+    int64_t
+    @endcode
+
+    #### Default behavior
+
+    - The restrictions about leading zeros is not enforced in C++. Instead,
+      leading zeros in integer literals lead to an interpretation as octal
+      number. Internally, the value will be stored as decimal number. For
+      instance, the C++ integer literal `010` will be serialized to `8`.
+      During deserialization, leading zeros yield an error.
+    - Not-a-number (NaN) values will be serialized to `null`.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the range and precision of numbers.
+
+    When the default type is used, the maximal integer number that can be
+    stored is `9223372036854775807` (INT64_MAX) and the minimal integer number
+    that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers
+    that are out of range will yield over/underflow when used in a
+    constructor. During deserialization, too large or small integer numbers
+    will be automatically be stored as @ref number_unsigned_t or @ref
+    number_float_t.
+
+    [RFC 7159](http://rfc7159.net/rfc7159) further states:
+    > Note that when such software is used, numbers that are integers and are
+    > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense
+    > that implementations will agree exactly on their numeric values.
+
+    As this range is a subrange of the exactly supported range [INT64_MIN,
+    INT64_MAX], this class's integer type is interoperable.
+
+    #### Storage
+
+    Integer number values are stored directly inside a @ref basic_json type.
+
+    @sa @ref number_float_t -- type for number values (floating-point)
+
+    @sa @ref number_unsigned_t -- type for number values (unsigned integer)
+
+    @since version 1.0.0
+    */
+    using number_integer_t = NumberIntegerType;
+
+    /*!
+    @brief a type for a number (unsigned)
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
+    > The representation of numbers is similar to that used in most
+    > programming languages. A number is represented in base 10 using decimal
+    > digits. It contains an integer component that may be prefixed with an
+    > optional minus sign, which may be followed by a fraction part and/or an
+    > exponent part. Leading zeros are not allowed. (...) Numeric values that
+    > cannot be represented in the grammar below (such as Infinity and NaN)
+    > are not permitted.
+
+    This description includes both integer and floating-point numbers.
+    However, C++ allows more precise storage if it is known whether the number
+    is a signed integer, an unsigned integer or a floating-point number.
+    Therefore, three different types, @ref number_integer_t, @ref
+    number_unsigned_t and @ref number_float_t are used.
+
+    To store unsigned integer numbers in C++, a type is defined by the
+    template parameter @a NumberUnsignedType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a NumberUnsignedType (`uint64_t`), the
+    default value for @a number_unsigned_t is:
+
+    @code {.cpp}
+    uint64_t
+    @endcode
+
+    #### Default behavior
+
+    - The restrictions about leading zeros is not enforced in C++. Instead,
+      leading zeros in integer literals lead to an interpretation as octal
+      number. Internally, the value will be stored as decimal number. For
+      instance, the C++ integer literal `010` will be serialized to `8`.
+      During deserialization, leading zeros yield an error.
+    - Not-a-number (NaN) values will be serialized to `null`.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the range and precision of numbers.
+
+    When the default type is used, the maximal integer number that can be
+    stored is `18446744073709551615` (UINT64_MAX) and the minimal integer
+    number that can be stored is `0`. Integer numbers that are out of range
+    will yield over/underflow when used in a constructor. During
+    deserialization, too large or small integer numbers will be automatically
+    be stored as @ref number_integer_t or @ref number_float_t.
+
+    [RFC 7159](http://rfc7159.net/rfc7159) further states:
+    > Note that when such software is used, numbers that are integers and are
+    > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense
+    > that implementations will agree exactly on their numeric values.
+
+    As this range is a subrange (when considered in conjunction with the
+    number_integer_t type) of the exactly supported range [0, UINT64_MAX],
+    this class's integer type is interoperable.
+
+    #### Storage
+
+    Integer number values are stored directly inside a @ref basic_json type.
+
+    @sa @ref number_float_t -- type for number values (floating-point)
+    @sa @ref number_integer_t -- type for number values (integer)
+
+    @since version 2.0.0
+    */
+    using number_unsigned_t = NumberUnsignedType;
+
+    /*!
+    @brief a type for a number (floating-point)
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
+    > The representation of numbers is similar to that used in most
+    > programming languages. A number is represented in base 10 using decimal
+    > digits. It contains an integer component that may be prefixed with an
+    > optional minus sign, which may be followed by a fraction part and/or an
+    > exponent part. Leading zeros are not allowed. (...) Numeric values that
+    > cannot be represented in the grammar below (such as Infinity and NaN)
+    > are not permitted.
+
+    This description includes both integer and floating-point numbers.
+    However, C++ allows more precise storage if it is known whether the number
+    is a signed integer, an unsigned integer or a floating-point number.
+    Therefore, three different types, @ref number_integer_t, @ref
+    number_unsigned_t and @ref number_float_t are used.
+
+    To store floating-point numbers in C++, a type is defined by the template
+    parameter @a NumberFloatType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a NumberFloatType (`double`), the default
+    value for @a number_float_t is:
+
+    @code {.cpp}
+    double
+    @endcode
+
+    #### Default behavior
+
+    - The restrictions about leading zeros is not enforced in C++. Instead,
+      leading zeros in floating-point literals will be ignored. Internally,
+      the value will be stored as decimal number. For instance, the C++
+      floating-point literal `01.2` will be serialized to `1.2`. During
+      deserialization, leading zeros yield an error.
+    - Not-a-number (NaN) values will be serialized to `null`.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) states:
+    > This specification allows implementations to set limits on the range and
+    > precision of numbers accepted. Since software that implements IEEE
+    > 754-2008 binary64 (double precision) numbers is generally available and
+    > widely used, good interoperability can be achieved by implementations
+    > that expect no more precision or range than these provide, in the sense
+    > that implementations will approximate JSON numbers within the expected
+    > precision.
+
+    This implementation does exactly follow this approach, as it uses double
+    precision floating-point numbers. Note values smaller than
+    `-1.79769313486232e+308` and values greater than `1.79769313486232e+308`
+    will be stored as NaN internally and be serialized to `null`.
+
+    #### Storage
+
+    Floating-point number values are stored directly inside a @ref basic_json
+    type.
+
+    @sa @ref number_integer_t -- type for number values (integer)
+
+    @sa @ref number_unsigned_t -- type for number values (unsigned integer)
+
+    @since version 1.0.0
+    */
+    using number_float_t = NumberFloatType;
+
+    /// @}
+
+  private:
+
+    /// helper for exception-safe object creation
+    template<typename T, typename... Args>
+    static T* create(Args&& ... args)
+    {
+        AllocatorType<T> alloc;
+        using AllocatorTraits = std::allocator_traits<AllocatorType<T>>;
+
+        auto deleter = [&](T * object)
+        {
+            AllocatorTraits::deallocate(alloc, object, 1);
+        };
+        std::unique_ptr<T, decltype(deleter)> object(AllocatorTraits::allocate(alloc, 1), deleter);
+        AllocatorTraits::construct(alloc, object.get(), std::forward<Args>(args)...);
+        assert(object != nullptr);
+        return object.release();
+    }
+
+    ////////////////////////
+    // JSON value storage //
+    ////////////////////////
+
+    /*!
+    @brief a JSON value
+
+    The actual storage for a JSON value of the @ref basic_json class. This
+    union combines the different storage types for the JSON value types
+    defined in @ref value_t.
+
+    JSON type | value_t type    | used type
+    --------- | --------------- | ------------------------
+    object    | object          | pointer to @ref object_t
+    array     | array           | pointer to @ref array_t
+    string    | string          | pointer to @ref string_t
+    boolean   | boolean         | @ref boolean_t
+    number    | number_integer  | @ref number_integer_t
+    number    | number_unsigned | @ref number_unsigned_t
+    number    | number_float    | @ref number_float_t
+    null      | null            | *no value is stored*
+
+    @note Variable-length types (objects, arrays, and strings) are stored as
+    pointers. The size of the union should not exceed 64 bits if the default
+    value types are used.
+
+    @since version 1.0.0
+    */
+    union json_value
+    {
+        /// object (stored with pointer to save storage)
+        object_t* object;
+        /// array (stored with pointer to save storage)
+        array_t* array;
+        /// string (stored with pointer to save storage)
+        string_t* string;
+        /// boolean
+        boolean_t boolean;
+        /// number (integer)
+        number_integer_t number_integer;
+        /// number (unsigned integer)
+        number_unsigned_t number_unsigned;
+        /// number (floating-point)
+        number_float_t number_float;
+
+        /// default constructor (for null values)
+        json_value() = default;
+        /// constructor for booleans
+        json_value(boolean_t v) noexcept : boolean(v) {}
+        /// constructor for numbers (integer)
+        json_value(number_integer_t v) noexcept : number_integer(v) {}
+        /// constructor for numbers (unsigned)
+        json_value(number_unsigned_t v) noexcept : number_unsigned(v) {}
+        /// constructor for numbers (floating-point)
+        json_value(number_float_t v) noexcept : number_float(v) {}
+        /// constructor for empty values of a given type
+        json_value(value_t t)
+        {
+            switch (t)
+            {
+                case value_t::object:
+                {
+                    object = create<object_t>();
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    array = create<array_t>();
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    string = create<string_t>("");
+                    break;
+                }
+
+                case value_t::boolean:
+                {
+                    boolean = boolean_t(false);
+                    break;
+                }
+
+                case value_t::number_integer:
+                {
+                    number_integer = number_integer_t(0);
+                    break;
+                }
+
+                case value_t::number_unsigned:
+                {
+                    number_unsigned = number_unsigned_t(0);
+                    break;
+                }
+
+                case value_t::number_float:
+                {
+                    number_float = number_float_t(0.0);
+                    break;
+                }
+
+                case value_t::null:
+                {
+                    object = nullptr;  // silence warning, see #821
+                    break;
+                }
+
+                default:
+                {
+                    object = nullptr;  // silence warning, see #821
+                    if (JSON_UNLIKELY(t == value_t::null))
+                    {
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE
+                    }
+                    break;
+                }
+            }
+        }
+
+        /// constructor for strings
+        json_value(const string_t& value)
+        {
+            string = create<string_t>(value);
+        }
+
+        /// constructor for rvalue strings
+        json_value(string_t&& value)
+        {
+            string = create<string_t>(std::move(value));
+        }
+
+        /// constructor for objects
+        json_value(const object_t& value)
+        {
+            object = create<object_t>(value);
+        }
+
+        /// constructor for rvalue objects
+        json_value(object_t&& value)
+        {
+            object = create<object_t>(std::move(value));
+        }
+
+        /// constructor for arrays
+        json_value(const array_t& value)
+        {
+            array = create<array_t>(value);
+        }
+
+        /// constructor for rvalue arrays
+        json_value(array_t&& value)
+        {
+            array = create<array_t>(std::move(value));
+        }
+
+        void destroy(value_t t) noexcept
+        {
+            switch (t)
+            {
+                case value_t::object:
+                {
+                    AllocatorType<object_t> alloc;
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, object);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1);
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    AllocatorType<array_t> alloc;
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, array);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1);
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    AllocatorType<string_t> alloc;
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, string);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1);
+                    break;
+                }
+
+                default:
+                {
+                    break;
+                }
+            }
+        }
+    };
+
+    /*!
+    @brief checks the class invariants
+
+    This function asserts the class invariants. It needs to be called at the
+    end of every constructor to make sure that created objects respect the
+    invariant. Furthermore, it has to be called each time the type of a JSON
+    value is changed, because the invariant expresses a relationship between
+    @a m_type and @a m_value.
+    */
+    void assert_invariant() const noexcept
+    {
+        assert(m_type != value_t::object or m_value.object != nullptr);
+        assert(m_type != value_t::array or m_value.array != nullptr);
+        assert(m_type != value_t::string or m_value.string != nullptr);
+    }
+
+  public:
+    //////////////////////////
+    // JSON parser callback //
+    //////////////////////////
+
+    /*!
+    @brief parser event types
+
+    The parser callback distinguishes the following events:
+    - `object_start`: the parser read `{` and started to process a JSON object
+    - `key`: the parser read a key of a value in an object
+    - `object_end`: the parser read `}` and finished processing a JSON object
+    - `array_start`: the parser read `[` and started to process a JSON array
+    - `array_end`: the parser read `]` and finished processing a JSON array
+    - `value`: the parser finished reading a JSON value
+
+    @image html callback_events.png "Example when certain parse events are triggered"
+
+    @sa @ref parser_callback_t for more information and examples
+    */
+    using parse_event_t = typename parser::parse_event_t;
+
+    /*!
+    @brief per-element parser callback type
+
+    With a parser callback function, the result of parsing a JSON text can be
+    influenced. When passed to @ref parse, it is called on certain events
+    (passed as @ref parse_event_t via parameter @a event) with a set recursion
+    depth @a depth and context JSON value @a parsed. The return value of the
+    callback function is a boolean indicating whether the element that emitted
+    the callback shall be kept or not.
+
+    We distinguish six scenarios (determined by the event type) in which the
+    callback function can be called. The following table describes the values
+    of the parameters @a depth, @a event, and @a parsed.
+
+    parameter @a event | description | parameter @a depth | parameter @a parsed
+    ------------------ | ----------- | ------------------ | -------------------
+    parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded
+    parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key
+    parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object
+    parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded
+    parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array
+    parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value
+
+    @image html callback_events.png "Example when certain parse events are triggered"
+
+    Discarding a value (i.e., returning `false`) has different effects
+    depending on the context in which function was called:
+
+    - Discarded values in structured types are skipped. That is, the parser
+      will behave as if the discarded value was never read.
+    - In case a value outside a structured type is skipped, it is replaced
+      with `null`. This case happens if the top-level element is skipped.
+
+    @param[in] depth  the depth of the recursion during parsing
+
+    @param[in] event  an event of type parse_event_t indicating the context in
+    the callback function has been called
+
+    @param[in,out] parsed  the current intermediate parse result; note that
+    writing to this value has no effect for parse_event_t::key events
+
+    @return Whether the JSON value which called the function during parsing
+    should be kept (`true`) or not (`false`). In the latter case, it is either
+    skipped completely or replaced by an empty discarded object.
+
+    @sa @ref parse for examples
+
+    @since version 1.0.0
+    */
+    using parser_callback_t = typename parser::parser_callback_t;
+
+    //////////////////
+    // constructors //
+    //////////////////
+
+    /// @name constructors and destructors
+    /// Constructors of class @ref basic_json, copy/move constructor, copy
+    /// assignment, static functions creating objects, and the destructor.
+    /// @{
+
+    /*!
+    @brief create an empty value with a given type
+
+    Create an empty JSON value with a given type. The value will be default
+    initialized with an empty value which depends on the type:
+
+    Value type  | initial value
+    ----------- | -------------
+    null        | `null`
+    boolean     | `false`
+    string      | `""`
+    number      | `0`
+    object      | `{}`
+    array       | `[]`
+
+    @param[in] v  the type of the value to create
+
+    @complexity Constant.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows the constructor for different @ref
+    value_t values,basic_json__value_t}
+
+    @sa @ref clear() -- restores the postcondition of this constructor
+
+    @since version 1.0.0
+    */
+    basic_json(const value_t v)
+        : m_type(v), m_value(v)
+    {
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a null object
+
+    Create a `null` JSON value. It either takes a null pointer as parameter
+    (explicitly creating `null`) or no parameter (implicitly creating `null`).
+    The passed null pointer itself is not read -- it is only used to choose
+    the right constructor.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this constructor never throws
+    exceptions.
+
+    @liveexample{The following code shows the constructor with and without a
+    null pointer parameter.,basic_json__nullptr_t}
+
+    @since version 1.0.0
+    */
+    basic_json(std::nullptr_t = nullptr) noexcept
+        : basic_json(value_t::null)
+    {
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a JSON value
+
+    This is a "catch all" constructor for all compatible JSON types; that is,
+    types for which a `to_json()` method exists. The constructor forwards the
+    parameter @a val to that method (to `json_serializer<U>::to_json` method
+    with `U = uncvref_t<CompatibleType>`, to be exact).
+
+    Template type @a CompatibleType includes, but is not limited to, the
+    following types:
+    - **arrays**: @ref array_t and all kinds of compatible containers such as
+      `std::vector`, `std::deque`, `std::list`, `std::forward_list`,
+      `std::array`, `std::valarray`, `std::set`, `std::unordered_set`,
+      `std::multiset`, and `std::unordered_multiset` with a `value_type` from
+      which a @ref basic_json value can be constructed.
+    - **objects**: @ref object_t and all kinds of compatible associative
+      containers such as `std::map`, `std::unordered_map`, `std::multimap`,
+      and `std::unordered_multimap` with a `key_type` compatible to
+      @ref string_t and a `value_type` from which a @ref basic_json value can
+      be constructed.
+    - **strings**: @ref string_t, string literals, and all compatible string
+      containers can be used.
+    - **numbers**: @ref number_integer_t, @ref number_unsigned_t,
+      @ref number_float_t, and all convertible number types such as `int`,
+      `size_t`, `int64_t`, `float` or `double` can be used.
+    - **boolean**: @ref boolean_t / `bool` can be used.
+
+    See the examples below.
+
+    @tparam CompatibleType a type such that:
+    - @a CompatibleType is not derived from `std::istream`,
+    - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move
+         constructors),
+    - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments)
+    - @a CompatibleType is not a @ref basic_json nested type (e.g.,
+         @ref json_pointer, @ref iterator, etc ...)
+    - @ref @ref json_serializer<U> has a
+         `to_json(basic_json_t&, CompatibleType&&)` method
+
+    @tparam U = `uncvref_t<CompatibleType>`
+
+    @param[in] val the value to be forwarded to the respective constructor
+
+    @complexity Usually linear in the size of the passed @a val, also
+                depending on the implementation of the called `to_json()`
+                method.
+
+    @exceptionsafety Depends on the called constructor. For types directly
+    supported by the library (i.e., all types for which no `to_json()` function
+    was provided), strong guarantee holds: if an exception is thrown, there are
+    no changes to any JSON value.
+
+    @liveexample{The following code shows the constructor with several
+    compatible types.,basic_json__CompatibleType}
+
+    @since version 2.1.0
+    */
+    template <typename CompatibleType,
+              typename U = detail::uncvref_t<CompatibleType>,
+              detail::enable_if_t<
+                  not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0>
+    basic_json(CompatibleType && val) noexcept(noexcept(
+                JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
+                                           std::forward<CompatibleType>(val))))
+    {
+        JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a JSON value from an existing one
+
+    This is a constructor for existing @ref basic_json types.
+    It does not hijack copy/move constructors, since the parameter has different
+    template arguments than the current ones.
+
+    The constructor tries to convert the internal @ref m_value of the parameter.
+
+    @tparam BasicJsonType a type such that:
+    - @a BasicJsonType is a @ref basic_json type.
+    - @a BasicJsonType has different template arguments than @ref basic_json_t.
+
+    @param[in] val the @ref basic_json value to be converted.
+
+    @complexity Usually linear in the size of the passed @a val, also
+                depending on the implementation of the called `to_json()`
+                method.
+
+    @exceptionsafety Depends on the called constructor. For types directly
+    supported by the library (i.e., all types for which no `to_json()` function
+    was provided), strong guarantee holds: if an exception is thrown, there are
+    no changes to any JSON value.
+
+    @since version 3.2.0
+    */
+    template <typename BasicJsonType,
+              detail::enable_if_t<
+                  detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0>
+    basic_json(const BasicJsonType& val)
+    {
+        using other_boolean_t = typename BasicJsonType::boolean_t;
+        using other_number_float_t = typename BasicJsonType::number_float_t;
+        using other_number_integer_t = typename BasicJsonType::number_integer_t;
+        using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+        using other_string_t = typename BasicJsonType::string_t;
+        using other_object_t = typename BasicJsonType::object_t;
+        using other_array_t = typename BasicJsonType::array_t;
+
+        switch (val.type())
+        {
+            case value_t::boolean:
+                JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>());
+                break;
+            case value_t::number_float:
+                JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>());
+                break;
+            case value_t::number_integer:
+                JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>());
+                break;
+            case value_t::number_unsigned:
+                JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>());
+                break;
+            case value_t::string:
+                JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>());
+                break;
+            case value_t::object:
+                JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>());
+                break;
+            case value_t::array:
+                JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>());
+                break;
+            case value_t::null:
+                *this = nullptr;
+                break;
+            case value_t::discarded:
+                m_type = value_t::discarded;
+                break;
+        }
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a container (array or object) from an initializer list
+
+    Creates a JSON value of type array or object from the passed initializer
+    list @a init. In case @a type_deduction is `true` (default), the type of
+    the JSON value to be created is deducted from the initializer list @a init
+    according to the following rules:
+
+    1. If the list is empty, an empty JSON object value `{}` is created.
+    2. If the list consists of pairs whose first element is a string, a JSON
+       object value is created where the first elements of the pairs are
+       treated as keys and the second elements are as values.
+    3. In all other cases, an array is created.
+
+    The rules aim to create the best fit between a C++ initializer list and
+    JSON values. The rationale is as follows:
+
+    1. The empty initializer list is written as `{}` which is exactly an empty
+       JSON object.
+    2. C++ has no way of describing mapped types other than to list a list of
+       pairs. As JSON requires that keys must be of type string, rule 2 is the
+       weakest constraint one can pose on initializer lists to interpret them
+       as an object.
+    3. In all other cases, the initializer list could not be interpreted as
+       JSON object type, so interpreting it as JSON array type is safe.
+
+    With the rules described above, the following JSON values cannot be
+    expressed by an initializer list:
+
+    - the empty array (`[]`): use @ref array(initializer_list_t)
+      with an empty initializer list in this case
+    - arrays whose elements satisfy rule 2: use @ref
+      array(initializer_list_t) with the same initializer list
+      in this case
+
+    @note When used without parentheses around an empty initializer list, @ref
+    basic_json() is called instead of this function, yielding the JSON null
+    value.
+
+    @param[in] init  initializer list with JSON values
+
+    @param[in] type_deduction internal parameter; when set to `true`, the type
+    of the JSON value is deducted from the initializer list @a init; when set
+    to `false`, the type provided via @a manual_type is forced. This mode is
+    used by the functions @ref array(initializer_list_t) and
+    @ref object(initializer_list_t).
+
+    @param[in] manual_type internal parameter; when @a type_deduction is set
+    to `false`, the created JSON value will use the provided type (only @ref
+    value_t::array and @ref value_t::object are valid); when @a type_deduction
+    is set to `true`, this parameter has no effect
+
+    @throw type_error.301 if @a type_deduction is `false`, @a manual_type is
+    `value_t::object`, but @a init contains an element which is not a pair
+    whose first element is a string. In this case, the constructor could not
+    create an object. If @a type_deduction would have be `true`, an array
+    would have been created. See @ref object(initializer_list_t)
+    for an example.
+
+    @complexity Linear in the size of the initializer list @a init.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The example below shows how JSON values are created from
+    initializer lists.,basic_json__list_init_t}
+
+    @sa @ref array(initializer_list_t) -- create a JSON array
+    value from an initializer list
+    @sa @ref object(initializer_list_t) -- create a JSON object
+    value from an initializer list
+
+    @since version 1.0.0
+    */
+    basic_json(initializer_list_t init,
+               bool type_deduction = true,
+               value_t manual_type = value_t::array)
+    {
+        // check if each element is an array with two elements whose first
+        // element is a string
+        bool is_an_object = std::all_of(init.begin(), init.end(),
+                                        [](const detail::json_ref<basic_json>& element_ref)
+        {
+            return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string());
+        });
+
+        // adjust type if type deduction is not wanted
+        if (not type_deduction)
+        {
+            // if array is wanted, do not create an object though possible
+            if (manual_type == value_t::array)
+            {
+                is_an_object = false;
+            }
+
+            // if object is wanted but impossible, throw an exception
+            if (JSON_UNLIKELY(manual_type == value_t::object and not is_an_object))
+            {
+                JSON_THROW(type_error::create(301, "cannot create object from initializer list"));
+            }
+        }
+
+        if (is_an_object)
+        {
+            // the initializer list is a list of pairs -> create object
+            m_type = value_t::object;
+            m_value = value_t::object;
+
+            std::for_each(init.begin(), init.end(), [this](const detail::json_ref<basic_json>& element_ref)
+            {
+                auto element = element_ref.moved_or_copied();
+                m_value.object->emplace(
+                    std::move(*((*element.m_value.array)[0].m_value.string)),
+                    std::move((*element.m_value.array)[1]));
+            });
+        }
+        else
+        {
+            // the initializer list describes an array -> create array
+            m_type = value_t::array;
+            m_value.array = create<array_t>(init.begin(), init.end());
+        }
+
+        assert_invariant();
+    }
+
+    /*!
+    @brief explicitly create an array from an initializer list
+
+    Creates a JSON array value from a given initializer list. That is, given a
+    list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the
+    initializer list is empty, the empty array `[]` is created.
+
+    @note This function is only needed to express two edge cases that cannot
+    be realized with the initializer list constructor (@ref
+    basic_json(initializer_list_t, bool, value_t)). These cases
+    are:
+    1. creating an array whose elements are all pairs whose first element is a
+    string -- in this case, the initializer list constructor would create an
+    object, taking the first elements as keys
+    2. creating an empty array -- passing the empty initializer list to the
+    initializer list constructor yields an empty object
+
+    @param[in] init  initializer list with JSON values to create an array from
+    (optional)
+
+    @return JSON array value
+
+    @complexity Linear in the size of @a init.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows an example for the `array`
+    function.,array}
+
+    @sa @ref basic_json(initializer_list_t, bool, value_t) --
+    create a JSON value from an initializer list
+    @sa @ref object(initializer_list_t) -- create a JSON object
+    value from an initializer list
+
+    @since version 1.0.0
+    */
+    static basic_json array(initializer_list_t init = {})
+    {
+        return basic_json(init, false, value_t::array);
+    }
+
+    /*!
+    @brief explicitly create an object from an initializer list
+
+    Creates a JSON object value from a given initializer list. The initializer
+    lists elements must be pairs, and their first elements must be strings. If
+    the initializer list is empty, the empty object `{}` is created.
+
+    @note This function is only added for symmetry reasons. In contrast to the
+    related function @ref array(initializer_list_t), there are
+    no cases which can only be expressed by this function. That is, any
+    initializer list @a init can also be passed to the initializer list
+    constructor @ref basic_json(initializer_list_t, bool, value_t).
+
+    @param[in] init  initializer list to create an object from (optional)
+
+    @return JSON object value
+
+    @throw type_error.301 if @a init is not a list of pairs whose first
+    elements are strings. In this case, no object can be created. When such a
+    value is passed to @ref basic_json(initializer_list_t, bool, value_t),
+    an array would have been created from the passed initializer list @a init.
+    See example below.
+
+    @complexity Linear in the size of @a init.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows an example for the `object`
+    function.,object}
+
+    @sa @ref basic_json(initializer_list_t, bool, value_t) --
+    create a JSON value from an initializer list
+    @sa @ref array(initializer_list_t) -- create a JSON array
+    value from an initializer list
+
+    @since version 1.0.0
+    */
+    static basic_json object(initializer_list_t init = {})
+    {
+        return basic_json(init, false, value_t::object);
+    }
+
+    /*!
+    @brief construct an array with count copies of given value
+
+    Constructs a JSON array value by creating @a cnt copies of a passed value.
+    In case @a cnt is `0`, an empty array is created.
+
+    @param[in] cnt  the number of JSON copies of @a val to create
+    @param[in] val  the JSON value to copy
+
+    @post `std::distance(begin(),end()) == cnt` holds.
+
+    @complexity Linear in @a cnt.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows examples for the @ref
+    basic_json(size_type\, const basic_json&)
+    constructor.,basic_json__size_type_basic_json}
+
+    @since version 1.0.0
+    */
+    basic_json(size_type cnt, const basic_json& val)
+        : m_type(value_t::array)
+    {
+        m_value.array = create<array_t>(cnt, val);
+        assert_invariant();
+    }
+
+    /*!
+    @brief construct a JSON container given an iterator range
+
+    Constructs the JSON value with the contents of the range `[first, last)`.
+    The semantics depends on the different types a JSON value can have:
+    - In case of a null type, invalid_iterator.206 is thrown.
+    - In case of other primitive types (number, boolean, or string), @a first
+      must be `begin()` and @a last must be `end()`. In this case, the value is
+      copied. Otherwise, invalid_iterator.204 is thrown.
+    - In case of structured types (array, object), the constructor behaves as
+      similar versions for `std::vector` or `std::map`; that is, a JSON array
+      or object is constructed from the values in the range.
+
+    @tparam InputIT an input iterator type (@ref iterator or @ref
+    const_iterator)
+
+    @param[in] first begin of the range to copy from (included)
+    @param[in] last end of the range to copy from (excluded)
+
+    @pre Iterators @a first and @a last must be initialized. **This
+         precondition is enforced with an assertion (see warning).** If
+         assertions are switched off, a violation of this precondition yields
+         undefined behavior.
+
+    @pre Range `[first, last)` is valid. Usually, this precondition cannot be
+         checked efficiently. Only certain edge cases are detected; see the
+         description of the exceptions below. A violation of this precondition
+         yields undefined behavior.
+
+    @warning A precondition is enforced with a runtime assertion that will
+             result in calling `std::abort` if this precondition is not met.
+             Assertions can be disabled by defining `NDEBUG` at compile time.
+             See https://en.cppreference.com/w/cpp/error/assert for more
+             information.
+
+    @throw invalid_iterator.201 if iterators @a first and @a last are not
+    compatible (i.e., do not belong to the same JSON value). In this case,
+    the range `[first, last)` is undefined.
+    @throw invalid_iterator.204 if iterators @a first and @a last belong to a
+    primitive type (number, boolean, or string), but @a first does not point
+    to the first element any more. In this case, the range `[first, last)` is
+    undefined. See example code below.
+    @throw invalid_iterator.206 if iterators @a first and @a last belong to a
+    null value. In this case, the range `[first, last)` is undefined.
+
+    @complexity Linear in distance between @a first and @a last.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The example below shows several ways to create JSON values by
+    specifying a subrange with iterators.,basic_json__InputIt_InputIt}
+
+    @since version 1.0.0
+    */
+    template<class InputIT, typename std::enable_if<
+                 std::is_same<InputIT, typename basic_json_t::iterator>::value or
+                 std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int>::type = 0>
+    basic_json(InputIT first, InputIT last)
+    {
+        assert(first.m_object != nullptr);
+        assert(last.m_object != nullptr);
+
+        // make sure iterator fits the current value
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(201, "iterators are not compatible"));
+        }
+
+        // copy type from first iterator
+        m_type = first.m_object->m_type;
+
+        // check if iterator range is complete for primitive values
+        switch (m_type)
+        {
+            case value_t::boolean:
+            case value_t::number_float:
+            case value_t::number_integer:
+            case value_t::number_unsigned:
+            case value_t::string:
+            {
+                if (JSON_UNLIKELY(not first.m_it.primitive_iterator.is_begin()
+                                  or not last.m_it.primitive_iterator.is_end()))
+                {
+                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+
+        switch (m_type)
+        {
+            case value_t::number_integer:
+            {
+                m_value.number_integer = first.m_object->m_value.number_integer;
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                m_value.number_unsigned = first.m_object->m_value.number_unsigned;
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                m_value.number_float = first.m_object->m_value.number_float;
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                m_value.boolean = first.m_object->m_value.boolean;
+                break;
+            }
+
+            case value_t::string:
+            {
+                m_value = *first.m_object->m_value.string;
+                break;
+            }
+
+            case value_t::object:
+            {
+                m_value.object = create<object_t>(first.m_it.object_iterator,
+                                                  last.m_it.object_iterator);
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_value.array = create<array_t>(first.m_it.array_iterator,
+                                                last.m_it.array_iterator);
+                break;
+            }
+
+            default:
+                JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " +
+                                                    std::string(first.m_object->type_name())));
+        }
+
+        assert_invariant();
+    }
+
+
+    ///////////////////////////////////////
+    // other constructors and destructor //
+    ///////////////////////////////////////
+
+    /// @private
+    basic_json(const detail::json_ref<basic_json>& ref)
+        : basic_json(ref.moved_or_copied())
+    {}
+
+    /*!
+    @brief copy constructor
+
+    Creates a copy of a given JSON value.
+
+    @param[in] other  the JSON value to copy
+
+    @post `*this == other`
+
+    @complexity Linear in the size of @a other.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is linear.
+    - As postcondition, it holds: `other == basic_json(other)`.
+
+    @liveexample{The following code shows an example for the copy
+    constructor.,basic_json__basic_json}
+
+    @since version 1.0.0
+    */
+    basic_json(const basic_json& other)
+        : m_type(other.m_type)
+    {
+        // check of passed value is valid
+        other.assert_invariant();
+
+        switch (m_type)
+        {
+            case value_t::object:
+            {
+                m_value = *other.m_value.object;
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_value = *other.m_value.array;
+                break;
+            }
+
+            case value_t::string:
+            {
+                m_value = *other.m_value.string;
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                m_value = other.m_value.boolean;
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                m_value = other.m_value.number_integer;
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                m_value = other.m_value.number_unsigned;
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                m_value = other.m_value.number_float;
+                break;
+            }
+
+            default:
+                break;
+        }
+
+        assert_invariant();
+    }
+
+    /*!
+    @brief move constructor
+
+    Move constructor. Constructs a JSON value with the contents of the given
+    value @a other using move semantics. It "steals" the resources from @a
+    other and leaves it as JSON null value.
+
+    @param[in,out] other  value to move to this object
+
+    @post `*this` has the same value as @a other before the call.
+    @post @a other is a JSON null value.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this constructor never throws
+    exceptions.
+
+    @requirement This function helps `basic_json` satisfying the
+    [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible)
+    requirements.
+
+    @liveexample{The code below shows the move constructor explicitly called
+    via std::move.,basic_json__moveconstructor}
+
+    @since version 1.0.0
+    */
+    basic_json(basic_json&& other) noexcept
+        : m_type(std::move(other.m_type)),
+          m_value(std::move(other.m_value))
+    {
+        // check that passed value is valid
+        other.assert_invariant();
+
+        // invalidate payload
+        other.m_type = value_t::null;
+        other.m_value = {};
+
+        assert_invariant();
+    }
+
+    /*!
+    @brief copy assignment
+
+    Copy assignment operator. Copies a JSON value via the "copy and swap"
+    strategy: It is expressed in terms of the copy constructor, destructor,
+    and the `swap()` member function.
+
+    @param[in] other  value to copy from
+
+    @complexity Linear.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is linear.
+
+    @liveexample{The code below shows and example for the copy assignment. It
+    creates a copy of value `a` which is then swapped with `b`. Finally\, the
+    copy of `a` (which is the null value after the swap) is
+    destroyed.,basic_json__copyassignment}
+
+    @since version 1.0.0
+    */
+    basic_json& operator=(basic_json other) noexcept (
+        std::is_nothrow_move_constructible<value_t>::value and
+        std::is_nothrow_move_assignable<value_t>::value and
+        std::is_nothrow_move_constructible<json_value>::value and
+        std::is_nothrow_move_assignable<json_value>::value
+    )
+    {
+        // check that passed value is valid
+        other.assert_invariant();
+
+        using std::swap;
+        swap(m_type, other.m_type);
+        swap(m_value, other.m_value);
+
+        assert_invariant();
+        return *this;
+    }
+
+    /*!
+    @brief destructor
+
+    Destroys the JSON value and frees all allocated memory.
+
+    @complexity Linear.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is linear.
+    - All stored elements are destroyed and all memory is freed.
+
+    @since version 1.0.0
+    */
+    ~basic_json() noexcept
+    {
+        assert_invariant();
+        m_value.destroy(m_type);
+    }
+
+    /// @}
+
+  public:
+    ///////////////////////
+    // object inspection //
+    ///////////////////////
+
+    /// @name object inspection
+    /// Functions to inspect the type of a JSON value.
+    /// @{
+
+    /*!
+    @brief serialization
+
+    Serialization function for JSON values. The function tries to mimic
+    Python's `json.dumps()` function, and currently supports its @a indent
+    and @a ensure_ascii parameters.
+
+    @param[in] indent If indent is nonnegative, then array elements and object
+    members will be pretty-printed with that indent level. An indent level of
+    `0` will only insert newlines. `-1` (the default) selects the most compact
+    representation.
+    @param[in] indent_char The character to use for indentation if @a indent is
+    greater than `0`. The default is ` ` (space).
+    @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
+    in the output are escaped with `\uXXXX` sequences, and the result consists
+    of ASCII characters only.
+    @param[in] error_handler  how to react on decoding errors; there are three
+    possible values: `strict` (throws and exception in case a decoding error
+    occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD),
+    and `ignore` (ignore invalid UTF-8 sequences during serialization).
+
+    @return string containing the serialization of the JSON value
+
+    @throw type_error.316 if a string stored inside the JSON value is not
+                          UTF-8 encoded
+
+    @complexity Linear.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @liveexample{The following example shows the effect of different @a indent\,
+    @a indent_char\, and @a ensure_ascii parameters to the result of the
+    serialization.,dump}
+
+    @see https://docs.python.org/2/library/json.html#json.dump
+
+    @since version 1.0.0; indentation character @a indent_char, option
+           @a ensure_ascii and exceptions added in version 3.0.0; error
+           handlers added in version 3.4.0.
+    */
+    string_t dump(const int indent = -1,
+                  const char indent_char = ' ',
+                  const bool ensure_ascii = false,
+                  const error_handler_t error_handler = error_handler_t::strict) const
+    {
+        string_t result;
+        serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
+
+        if (indent >= 0)
+        {
+            s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));
+        }
+        else
+        {
+            s.dump(*this, false, ensure_ascii, 0);
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief return the type of the JSON value (explicit)
+
+    Return the type of the JSON value as a value from the @ref value_t
+    enumeration.
+
+    @return the type of the JSON value
+            Value type                | return value
+            ------------------------- | -------------------------
+            null                      | value_t::null
+            boolean                   | value_t::boolean
+            string                    | value_t::string
+            number (integer)          | value_t::number_integer
+            number (unsigned integer) | value_t::number_unsigned
+            number (floating-point)   | value_t::number_float
+            object                    | value_t::object
+            array                     | value_t::array
+            discarded                 | value_t::discarded
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `type()` for all JSON
+    types.,type}
+
+    @sa @ref operator value_t() -- return the type of the JSON value (implicit)
+    @sa @ref type_name() -- return the type as string
+
+    @since version 1.0.0
+    */
+    constexpr value_t type() const noexcept
+    {
+        return m_type;
+    }
+
+    /*!
+    @brief return whether type is primitive
+
+    This function returns true if and only if the JSON type is primitive
+    (string, number, boolean, or null).
+
+    @return `true` if type is primitive (string, number, boolean, or null),
+    `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_primitive()` for all JSON
+    types.,is_primitive}
+
+    @sa @ref is_structured() -- returns whether JSON value is structured
+    @sa @ref is_null() -- returns whether JSON value is `null`
+    @sa @ref is_string() -- returns whether JSON value is a string
+    @sa @ref is_boolean() -- returns whether JSON value is a boolean
+    @sa @ref is_number() -- returns whether JSON value is a number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_primitive() const noexcept
+    {
+        return is_null() or is_string() or is_boolean() or is_number();
+    }
+
+    /*!
+    @brief return whether type is structured
+
+    This function returns true if and only if the JSON type is structured
+    (array or object).
+
+    @return `true` if type is structured (array or object), `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_structured()` for all JSON
+    types.,is_structured}
+
+    @sa @ref is_primitive() -- returns whether value is primitive
+    @sa @ref is_array() -- returns whether value is an array
+    @sa @ref is_object() -- returns whether value is an object
+
+    @since version 1.0.0
+    */
+    constexpr bool is_structured() const noexcept
+    {
+        return is_array() or is_object();
+    }
+
+    /*!
+    @brief return whether value is null
+
+    This function returns true if and only if the JSON value is null.
+
+    @return `true` if type is null, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_null()` for all JSON
+    types.,is_null}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_null() const noexcept
+    {
+        return (m_type == value_t::null);
+    }
+
+    /*!
+    @brief return whether value is a boolean
+
+    This function returns true if and only if the JSON value is a boolean.
+
+    @return `true` if type is boolean, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_boolean()` for all JSON
+    types.,is_boolean}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_boolean() const noexcept
+    {
+        return (m_type == value_t::boolean);
+    }
+
+    /*!
+    @brief return whether value is a number
+
+    This function returns true if and only if the JSON value is a number. This
+    includes both integer (signed and unsigned) and floating-point values.
+
+    @return `true` if type is number (regardless whether integer, unsigned
+    integer or floating-type), `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number()` for all JSON
+    types.,is_number}
+
+    @sa @ref is_number_integer() -- check if value is an integer or unsigned
+    integer number
+    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
+    number
+    @sa @ref is_number_float() -- check if value is a floating-point number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_number() const noexcept
+    {
+        return is_number_integer() or is_number_float();
+    }
+
+    /*!
+    @brief return whether value is an integer number
+
+    This function returns true if and only if the JSON value is a signed or
+    unsigned integer number. This excludes floating-point values.
+
+    @return `true` if type is an integer or unsigned integer number, `false`
+    otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number_integer()` for all
+    JSON types.,is_number_integer}
+
+    @sa @ref is_number() -- check if value is a number
+    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
+    number
+    @sa @ref is_number_float() -- check if value is a floating-point number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_number_integer() const noexcept
+    {
+        return (m_type == value_t::number_integer or m_type == value_t::number_unsigned);
+    }
+
+    /*!
+    @brief return whether value is an unsigned integer number
+
+    This function returns true if and only if the JSON value is an unsigned
+    integer number. This excludes floating-point and signed integer values.
+
+    @return `true` if type is an unsigned integer number, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number_unsigned()` for all
+    JSON types.,is_number_unsigned}
+
+    @sa @ref is_number() -- check if value is a number
+    @sa @ref is_number_integer() -- check if value is an integer or unsigned
+    integer number
+    @sa @ref is_number_float() -- check if value is a floating-point number
+
+    @since version 2.0.0
+    */
+    constexpr bool is_number_unsigned() const noexcept
+    {
+        return (m_type == value_t::number_unsigned);
+    }
+
+    /*!
+    @brief return whether value is a floating-point number
+
+    This function returns true if and only if the JSON value is a
+    floating-point number. This excludes signed and unsigned integer values.
+
+    @return `true` if type is a floating-point number, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number_float()` for all
+    JSON types.,is_number_float}
+
+    @sa @ref is_number() -- check if value is number
+    @sa @ref is_number_integer() -- check if value is an integer number
+    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
+    number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_number_float() const noexcept
+    {
+        return (m_type == value_t::number_float);
+    }
+
+    /*!
+    @brief return whether value is an object
+
+    This function returns true if and only if the JSON value is an object.
+
+    @return `true` if type is object, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_object()` for all JSON
+    types.,is_object}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_object() const noexcept
+    {
+        return (m_type == value_t::object);
+    }
+
+    /*!
+    @brief return whether value is an array
+
+    This function returns true if and only if the JSON value is an array.
+
+    @return `true` if type is array, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_array()` for all JSON
+    types.,is_array}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_array() const noexcept
+    {
+        return (m_type == value_t::array);
+    }
+
+    /*!
+    @brief return whether value is a string
+
+    This function returns true if and only if the JSON value is a string.
+
+    @return `true` if type is string, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_string()` for all JSON
+    types.,is_string}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_string() const noexcept
+    {
+        return (m_type == value_t::string);
+    }
+
+    /*!
+    @brief return whether value is discarded
+
+    This function returns true if and only if the JSON value was discarded
+    during parsing with a callback function (see @ref parser_callback_t).
+
+    @note This function will always be `false` for JSON values after parsing.
+    That is, discarded values can only occur during parsing, but will be
+    removed when inside a structured value or replaced by null in other cases.
+
+    @return `true` if type is discarded, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_discarded()` for all JSON
+    types.,is_discarded}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_discarded() const noexcept
+    {
+        return (m_type == value_t::discarded);
+    }
+
+    /*!
+    @brief return the type of the JSON value (implicit)
+
+    Implicitly return the type of the JSON value as a value from the @ref
+    value_t enumeration.
+
+    @return the type of the JSON value
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies the @ref value_t operator for
+    all JSON types.,operator__value_t}
+
+    @sa @ref type() -- return the type of the JSON value (explicit)
+    @sa @ref type_name() -- return the type as string
+
+    @since version 1.0.0
+    */
+    constexpr operator value_t() const noexcept
+    {
+        return m_type;
+    }
+
+    /// @}
+
+  private:
+    //////////////////
+    // value access //
+    //////////////////
+
+    /// get a boolean (explicit)
+    boolean_t get_impl(boolean_t* /*unused*/) const
+    {
+        if (JSON_LIKELY(is_boolean()))
+        {
+            return m_value.boolean;
+        }
+
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name())));
+    }
+
+    /// get a pointer to the value (object)
+    object_t* get_impl_ptr(object_t* /*unused*/) noexcept
+    {
+        return is_object() ? m_value.object : nullptr;
+    }
+
+    /// get a pointer to the value (object)
+    constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept
+    {
+        return is_object() ? m_value.object : nullptr;
+    }
+
+    /// get a pointer to the value (array)
+    array_t* get_impl_ptr(array_t* /*unused*/) noexcept
+    {
+        return is_array() ? m_value.array : nullptr;
+    }
+
+    /// get a pointer to the value (array)
+    constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept
+    {
+        return is_array() ? m_value.array : nullptr;
+    }
+
+    /// get a pointer to the value (string)
+    string_t* get_impl_ptr(string_t* /*unused*/) noexcept
+    {
+        return is_string() ? m_value.string : nullptr;
+    }
+
+    /// get a pointer to the value (string)
+    constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept
+    {
+        return is_string() ? m_value.string : nullptr;
+    }
+
+    /// get a pointer to the value (boolean)
+    boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept
+    {
+        return is_boolean() ? &m_value.boolean : nullptr;
+    }
+
+    /// get a pointer to the value (boolean)
+    constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept
+    {
+        return is_boolean() ? &m_value.boolean : nullptr;
+    }
+
+    /// get a pointer to the value (integer number)
+    number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept
+    {
+        return is_number_integer() ? &m_value.number_integer : nullptr;
+    }
+
+    /// get a pointer to the value (integer number)
+    constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept
+    {
+        return is_number_integer() ? &m_value.number_integer : nullptr;
+    }
+
+    /// get a pointer to the value (unsigned number)
+    number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept
+    {
+        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
+    }
+
+    /// get a pointer to the value (unsigned number)
+    constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept
+    {
+        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
+    }
+
+    /// get a pointer to the value (floating-point number)
+    number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept
+    {
+        return is_number_float() ? &m_value.number_float : nullptr;
+    }
+
+    /// get a pointer to the value (floating-point number)
+    constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept
+    {
+        return is_number_float() ? &m_value.number_float : nullptr;
+    }
+
+    /*!
+    @brief helper function to implement get_ref()
+
+    This function helps to implement get_ref() without code duplication for
+    const and non-const overloads
+
+    @tparam ThisType will be deduced as `basic_json` or `const basic_json`
+
+    @throw type_error.303 if ReferenceType does not match underlying value
+    type of the current JSON
+    */
+    template<typename ReferenceType, typename ThisType>
+    static ReferenceType get_ref_impl(ThisType& obj)
+    {
+        // delegate the call to get_ptr<>()
+        auto ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>();
+
+        if (JSON_LIKELY(ptr != nullptr))
+        {
+            return *ptr;
+        }
+
+        JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name())));
+    }
+
+  public:
+    /// @name value access
+    /// Direct access to the stored value of a JSON value.
+    /// @{
+
+    /*!
+    @brief get special-case overload
+
+    This overloads avoids a lot of template boilerplate, it can be seen as the
+    identity method
+
+    @tparam BasicJsonType == @ref basic_json
+
+    @return a copy of *this
+
+    @complexity Constant.
+
+    @since version 2.1.0
+    */
+    template<typename BasicJsonType, detail::enable_if_t<
+                 std::is_same<typename std::remove_const<BasicJsonType>::type, basic_json_t>::value,
+                 int> = 0>
+    basic_json get() const
+    {
+        return *this;
+    }
+
+    /*!
+    @brief get special-case overload
+
+    This overloads converts the current @ref basic_json in a different
+    @ref basic_json type
+
+    @tparam BasicJsonType == @ref basic_json
+
+    @return a copy of *this, converted into @tparam BasicJsonType
+
+    @complexity Depending on the implementation of the called `from_json()`
+                method.
+
+    @since version 3.2.0
+    */
+    template<typename BasicJsonType, detail::enable_if_t<
+                 not std::is_same<BasicJsonType, basic_json>::value and
+                 detail::is_basic_json<BasicJsonType>::value, int> = 0>
+    BasicJsonType get() const
+    {
+        return *this;
+    }
+
+    /*!
+    @brief get a value (explicit)
+
+    Explicit type conversion between the JSON value and a compatible value
+    which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+    and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
+    The value is converted by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    ValueType ret;
+    JSONSerializer<ValueType>::from_json(*this, ret);
+    return ret;
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json,
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `void from_json(const basic_json&, ValueType&)`, and
+    - @ref json_serializer<ValueType> does not have a `from_json()` method of
+      the form `ValueType from_json(const basic_json&)`
+
+    @tparam ValueTypeCV the provided value type
+    @tparam ValueType the returned value type
+
+    @return copy of the JSON value, converted to @a ValueType
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,get__ValueType_const}
+
+    @since version 2.1.0
+    */
+    template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
+             detail::enable_if_t <
+                 not detail::is_basic_json<ValueType>::value and
+                 detail::has_from_json<basic_json_t, ValueType>::value and
+                 not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+                 int> = 0>
+    ValueType get() const noexcept(noexcept(
+                                       JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))
+    {
+        // we cannot static_assert on ValueTypeCV being non-const, because
+        // there is support for get<const basic_json_t>(), which is why we
+        // still need the uncvref
+        static_assert(not std::is_reference<ValueTypeCV>::value,
+                      "get() cannot be used with reference types, you might want to use get_ref()");
+        static_assert(std::is_default_constructible<ValueType>::value,
+                      "types must be DefaultConstructible when used with get()");
+
+        ValueType ret;
+        JSONSerializer<ValueType>::from_json(*this, ret);
+        return ret;
+    }
+
+    /*!
+    @brief get a value (explicit); special case
+
+    Explicit type conversion between the JSON value and a compatible value
+    which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+    and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
+    The value is converted by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    return JSONSerializer<ValueTypeCV>::from_json(*this);
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json and
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `ValueType from_json(const basic_json&)`
+
+    @note If @ref json_serializer<ValueType> has both overloads of
+    `from_json()`, this one is chosen.
+
+    @tparam ValueTypeCV the provided value type
+    @tparam ValueType the returned value type
+
+    @return copy of the JSON value, converted to @a ValueType
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @since version 2.1.0
+    */
+    template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
+             detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and
+                                 detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+                                 int> = 0>
+    ValueType get() const noexcept(noexcept(
+                                       JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>())))
+    {
+        static_assert(not std::is_reference<ValueTypeCV>::value,
+                      "get() cannot be used with reference types, you might want to use get_ref()");
+        return JSONSerializer<ValueTypeCV>::from_json(*this);
+    }
+
+    /*!
+    @brief get a value (explicit)
+
+    Explicit type conversion between the JSON value and a compatible value.
+    The value is filled into the input parameter by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    ValueType v;
+    JSONSerializer<ValueType>::from_json(*this, v);
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json,
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `void from_json(const basic_json&, ValueType&)`, and
+
+    @tparam ValueType the input parameter type.
+
+    @return the input parameter, allowing chaining calls.
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,get_to}
+
+    @since version 3.3.0
+    */
+    template<typename ValueType,
+             detail::enable_if_t <
+                 not detail::is_basic_json<ValueType>::value and
+                 detail::has_from_json<basic_json_t, ValueType>::value,
+                 int> = 0>
+    ValueType & get_to(ValueType& v) const noexcept(noexcept(
+                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v)))
+    {
+        JSONSerializer<ValueType>::from_json(*this, v);
+        return v;
+    }
+
+
+    /*!
+    @brief get a pointer value (implicit)
+
+    Implicit pointer access to the internally stored JSON value. No copies are
+    made.
+
+    @warning Writing data to the pointee of the result yields an undefined
+    state.
+
+    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
+    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
+    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
+    assertion.
+
+    @return pointer to the internally stored JSON value if the requested
+    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how pointers to internal values of a
+    JSON value can be requested. Note that no type conversions are made and a
+    `nullptr` is returned if the value and the requested pointer type does not
+    match.,get_ptr}
+
+    @since version 1.0.0
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
+    {
+        // delegate the call to get_impl_ptr<>()
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
+    }
+
+    /*!
+    @brief get a pointer value (implicit)
+    @copydoc get_ptr()
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value and
+                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
+    constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
+    {
+        // delegate the call to get_impl_ptr<>() const
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
+    }
+
+    /*!
+    @brief get a pointer value (explicit)
+
+    Explicit pointer access to the internally stored JSON value. No copies are
+    made.
+
+    @warning The pointer becomes invalid if the underlying JSON object
+    changes.
+
+    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
+    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
+    @ref number_unsigned_t, or @ref number_float_t.
+
+    @return pointer to the internally stored JSON value if the requested
+    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how pointers to internal values of a
+    JSON value can be requested. Note that no type conversions are made and a
+    `nullptr` is returned if the value and the requested pointer type does not
+    match.,get__PointerType}
+
+    @sa @ref get_ptr() for explicit pointer-member access
+
+    @since version 1.0.0
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>())
+    {
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
+    }
+
+    /*!
+    @brief get a pointer value (explicit)
+    @copydoc get()
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>())
+    {
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
+    }
+
+    /*!
+    @brief get a reference value (implicit)
+
+    Implicit reference access to the internally stored JSON value. No copies
+    are made.
+
+    @warning Writing data to the referee of the result yields an undefined
+    state.
+
+    @tparam ReferenceType reference type; must be a reference to @ref array_t,
+    @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or
+    @ref number_float_t. Enforced by static assertion.
+
+    @return reference to the internally stored JSON value if the requested
+    reference type @a ReferenceType fits to the JSON value; throws
+    type_error.303 otherwise
+
+    @throw type_error.303 in case passed type @a ReferenceType is incompatible
+    with the stored JSON value; see example below
+
+    @complexity Constant.
+
+    @liveexample{The example shows several calls to `get_ref()`.,get_ref}
+
+    @since version 1.1.0
+    */
+    template<typename ReferenceType, typename std::enable_if<
+                 std::is_reference<ReferenceType>::value, int>::type = 0>
+    ReferenceType get_ref()
+    {
+        // delegate call to get_ref_impl
+        return get_ref_impl<ReferenceType>(*this);
+    }
+
+    /*!
+    @brief get a reference value (implicit)
+    @copydoc get_ref()
+    */
+    template<typename ReferenceType, typename std::enable_if<
+                 std::is_reference<ReferenceType>::value and
+                 std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int>::type = 0>
+    ReferenceType get_ref() const
+    {
+        // delegate call to get_ref_impl
+        return get_ref_impl<ReferenceType>(*this);
+    }
+
+    /*!
+    @brief get a value (implicit)
+
+    Implicit type conversion between the JSON value and a compatible value.
+    The call is realized by calling @ref get() const.
+
+    @tparam ValueType non-pointer type compatible to the JSON value, for
+    instance `int` for JSON integer numbers, `bool` for JSON booleans, or
+    `std::vector` types for JSON arrays. The character type of @ref string_t
+    as well as an initializer list of this type is excluded to avoid
+    ambiguities as these types implicitly convert to `std::string`.
+
+    @return copy of the JSON value, converted to type @a ValueType
+
+    @throw type_error.302 in case passed type @a ValueType is incompatible
+    to the JSON value type (e.g., the JSON value is of type boolean, but a
+    string is requested); see example below
+
+    @complexity Linear in the size of the JSON value.
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,operator__ValueType}
+
+    @since version 1.0.0
+    */
+    template < typename ValueType, typename std::enable_if <
+                   not std::is_pointer<ValueType>::value and
+                   not std::is_same<ValueType, detail::json_ref<basic_json>>::value and
+                   not std::is_same<ValueType, typename string_t::value_type>::value and
+                   not detail::is_basic_json<ValueType>::value
+
+#ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
+                   and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
+#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914
+                   and not std::is_same<ValueType, typename std::string_view>::value
+#endif
+#endif
+                   and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value
+                   , int >::type = 0 >
+    operator ValueType() const
+    {
+        // delegate the call to get<>() const
+        return get<ValueType>();
+    }
+
+    /// @}
+
+
+    ////////////////////
+    // element access //
+    ////////////////////
+
+    /// @name element access
+    /// Access to the JSON value.
+    /// @{
+
+    /*!
+    @brief access specified array element with bounds checking
+
+    Returns a reference to the element at specified location @a idx, with
+    bounds checking.
+
+    @param[in] idx  index of the element to access
+
+    @return reference to the element at index @a idx
+
+    @throw type_error.304 if the JSON value is not an array; in this case,
+    calling `at` with an index makes no sense. See example below.
+    @throw out_of_range.401 if the index @a idx is out of range of the array;
+    that is, `idx >= size()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how array elements can be read and
+    written using `at()`. It also demonstrates the different exceptions that
+    can be thrown.,at__size_type}
+    */
+    reference at(size_type idx)
+    {
+        // at only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            JSON_TRY
+            {
+                return m_value.array->at(idx);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified array element with bounds checking
+
+    Returns a const reference to the element at specified location @a idx,
+    with bounds checking.
+
+    @param[in] idx  index of the element to access
+
+    @return const reference to the element at index @a idx
+
+    @throw type_error.304 if the JSON value is not an array; in this case,
+    calling `at` with an index makes no sense. See example below.
+    @throw out_of_range.401 if the index @a idx is out of range of the array;
+    that is, `idx >= size()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how array elements can be read using
+    `at()`. It also demonstrates the different exceptions that can be thrown.,
+    at__size_type_const}
+    */
+    const_reference at(size_type idx) const
+    {
+        // at only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            JSON_TRY
+            {
+                return m_value.array->at(idx);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified object element with bounds checking
+
+    Returns a reference to the element at with specified key @a key, with
+    bounds checking.
+
+    @param[in] key  key of the element to access
+
+    @return reference to the element at key @a key
+
+    @throw type_error.304 if the JSON value is not an object; in this case,
+    calling `at` with a key makes no sense. See example below.
+    @throw out_of_range.403 if the key @a key is is not stored in the object;
+    that is, `find(key) == end()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Logarithmic in the size of the container.
+
+    @sa @ref operator[](const typename object_t::key_type&) for unchecked
+    access by reference
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how object elements can be read and
+    written using `at()`. It also demonstrates the different exceptions that
+    can be thrown.,at__object_t_key_type}
+    */
+    reference at(const typename object_t::key_type& key)
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            JSON_TRY
+            {
+                return m_value.object->at(key);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified object element with bounds checking
+
+    Returns a const reference to the element at with specified key @a key,
+    with bounds checking.
+
+    @param[in] key  key of the element to access
+
+    @return const reference to the element at key @a key
+
+    @throw type_error.304 if the JSON value is not an object; in this case,
+    calling `at` with a key makes no sense. See example below.
+    @throw out_of_range.403 if the key @a key is is not stored in the object;
+    that is, `find(key) == end()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Logarithmic in the size of the container.
+
+    @sa @ref operator[](const typename object_t::key_type&) for unchecked
+    access by reference
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how object elements can be read using
+    `at()`. It also demonstrates the different exceptions that can be thrown.,
+    at__object_t_key_type_const}
+    */
+    const_reference at(const typename object_t::key_type& key) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            JSON_TRY
+            {
+                return m_value.object->at(key);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified array element
+
+    Returns a reference to the element at specified location @a idx.
+
+    @note If @a idx is beyond the range of the array (i.e., `idx >= size()`),
+    then the array is silently filled up with `null` values to make `idx` a
+    valid reference to the last stored element.
+
+    @param[in] idx  index of the element to access
+
+    @return reference to the element at index @a idx
+
+    @throw type_error.305 if the JSON value is not an array or null; in that
+    cases, using the [] operator with an index makes no sense.
+
+    @complexity Constant if @a idx is in the range of the array. Otherwise
+    linear in `idx - size()`.
+
+    @liveexample{The example below shows how array elements can be read and
+    written using `[]` operator. Note the addition of `null`
+    values.,operatorarray__size_type}
+
+    @since version 1.0.0
+    */
+    reference operator[](size_type idx)
+    {
+        // implicitly convert null value to an empty array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value.array = create<array_t>();
+            assert_invariant();
+        }
+
+        // operator[] only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            // fill up array with null values if given idx is outside range
+            if (idx >= m_value.array->size())
+            {
+                m_value.array->insert(m_value.array->end(),
+                                      idx - m_value.array->size() + 1,
+                                      basic_json());
+            }
+
+            return m_value.array->operator[](idx);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified array element
+
+    Returns a const reference to the element at specified location @a idx.
+
+    @param[in] idx  index of the element to access
+
+    @return const reference to the element at index @a idx
+
+    @throw type_error.305 if the JSON value is not an array; in that case,
+    using the [] operator with an index makes no sense.
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how array elements can be read using
+    the `[]` operator.,operatorarray__size_type_const}
+
+    @since version 1.0.0
+    */
+    const_reference operator[](size_type idx) const
+    {
+        // const operator[] only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            return m_value.array->operator[](idx);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified object element
+
+    Returns a reference to the element at with specified key @a key.
+
+    @note If @a key is not found in the object, then it is silently added to
+    the object and filled with a `null` value to make `key` a valid reference.
+    In case the value was `null` before, it is converted to an object.
+
+    @param[in] key  key of the element to access
+
+    @return reference to the element at key @a key
+
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read and
+    written using the `[]` operator.,operatorarray__key_type}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+    */
+    reference operator[](const typename object_t::key_type& key)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        // operator[] only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            return m_value.object->operator[](key);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief read-only access specified object element
+
+    Returns a const reference to the element at with specified key @a key. No
+    bounds checking is performed.
+
+    @warning If the element with key @a key does not exist, the behavior is
+    undefined.
+
+    @param[in] key  key of the element to access
+
+    @return const reference to the element at key @a key
+
+    @pre The element with key @a key must exist. **This precondition is
+         enforced with an assertion.**
+
+    @throw type_error.305 if the JSON value is not an object; in that case,
+    using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read using
+    the `[]` operator.,operatorarray__key_type_const}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+    */
+    const_reference operator[](const typename object_t::key_type& key) const
+    {
+        // const operator[] only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            assert(m_value.object->find(key) != m_value.object->end());
+            return m_value.object->find(key)->second;
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified object element
+
+    Returns a reference to the element at with specified key @a key.
+
+    @note If @a key is not found in the object, then it is silently added to
+    the object and filled with a `null` value to make `key` a valid reference.
+    In case the value was `null` before, it is converted to an object.
+
+    @param[in] key  key of the element to access
+
+    @return reference to the element at key @a key
+
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read and
+    written using the `[]` operator.,operatorarray__key_type}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.1.0
+    */
+    template<typename T>
+    reference operator[](T* key)
+    {
+        // implicitly convert null to object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value = value_t::object;
+            assert_invariant();
+        }
+
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            return m_value.object->operator[](key);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief read-only access specified object element
+
+    Returns a const reference to the element at with specified key @a key. No
+    bounds checking is performed.
+
+    @warning If the element with key @a key does not exist, the behavior is
+    undefined.
+
+    @param[in] key  key of the element to access
+
+    @return const reference to the element at key @a key
+
+    @pre The element with key @a key must exist. **This precondition is
+         enforced with an assertion.**
+
+    @throw type_error.305 if the JSON value is not an object; in that case,
+    using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read using
+    the `[]` operator.,operatorarray__key_type_const}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.1.0
+    */
+    template<typename T>
+    const_reference operator[](T* key) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            assert(m_value.object->find(key) != m_value.object->end());
+            return m_value.object->find(key)->second;
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified object element with default value
+
+    Returns either a copy of an object's element at the specified key @a key
+    or a given default value if no element with key @a key exists.
+
+    The function is basically equivalent to executing
+    @code {.cpp}
+    try {
+        return at(key);
+    } catch(out_of_range) {
+        return default_value;
+    }
+    @endcode
+
+    @note Unlike @ref at(const typename object_t::key_type&), this function
+    does not throw if the given key @a key was not found.
+
+    @note Unlike @ref operator[](const typename object_t::key_type& key), this
+    function does not implicitly add an element to the position defined by @a
+    key. This function is furthermore also applicable to const objects.
+
+    @param[in] key  key of the element to access
+    @param[in] default_value  the value to return if @a key is not found
+
+    @tparam ValueType type compatible to JSON values, for instance `int` for
+    JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for
+    JSON arrays. Note the type of the expected value at @a key and the default
+    value @a default_value must be compatible.
+
+    @return copy of the element at key @a key or @a default_value if @a key
+    is not found
+
+    @throw type_error.306 if the JSON value is not an object; in that case,
+    using `value()` with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be queried
+    with a default value.,basic_json__value}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref operator[](const typename object_t::key_type&) for unchecked
+    access by reference
+
+    @since version 1.0.0
+    */
+    template<class ValueType, typename std::enable_if<
+                 std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
+    ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            // if key is found, return value and given default value otherwise
+            const auto it = find(key);
+            if (it != end())
+            {
+                return *it;
+            }
+
+            return default_value;
+        }
+
+        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief overload for a default value of type const char*
+    @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const
+    */
+    string_t value(const typename object_t::key_type& key, const char* default_value) const
+    {
+        return value(key, string_t(default_value));
+    }
+
+    /*!
+    @brief access specified object element via JSON Pointer with default value
+
+    Returns either a copy of an object's element at the specified key @a key
+    or a given default value if no element with key @a key exists.
+
+    The function is basically equivalent to executing
+    @code {.cpp}
+    try {
+        return at(ptr);
+    } catch(out_of_range) {
+        return default_value;
+    }
+    @endcode
+
+    @note Unlike @ref at(const json_pointer&), this function does not throw
+    if the given key @a key was not found.
+
+    @param[in] ptr  a JSON pointer to the element to access
+    @param[in] default_value  the value to return if @a ptr found no value
+
+    @tparam ValueType type compatible to JSON values, for instance `int` for
+    JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for
+    JSON arrays. Note the type of the expected value at @a key and the default
+    value @a default_value must be compatible.
+
+    @return copy of the element at key @a key or @a default_value if @a key
+    is not found
+
+    @throw type_error.306 if the JSON value is not an object; in that case,
+    using `value()` with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be queried
+    with a default value.,basic_json__value_ptr}
+
+    @sa @ref operator[](const json_pointer&) for unchecked access by reference
+
+    @since version 2.0.2
+    */
+    template<class ValueType, typename std::enable_if<
+                 std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
+    ValueType value(const json_pointer& ptr, const ValueType& default_value) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            // if pointer resolves a value, return it or use default value
+            JSON_TRY
+            {
+                return ptr.get_checked(this);
+            }
+            JSON_INTERNAL_CATCH (out_of_range&)
+            {
+                return default_value;
+            }
+        }
+
+        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief overload for a default value of type const char*
+    @copydoc basic_json::value(const json_pointer&, ValueType) const
+    */
+    string_t value(const json_pointer& ptr, const char* default_value) const
+    {
+        return value(ptr, string_t(default_value));
+    }
+
+    /*!
+    @brief access the first element
+
+    Returns a reference to the first element in the container. For a JSON
+    container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
+
+    @return In case of a structured type (array or object), a reference to the
+    first element is returned. In case of number, string, or boolean values, a
+    reference to the value is returned.
+
+    @complexity Constant.
+
+    @pre The JSON value must not be `null` (would throw `std::out_of_range`)
+    or an empty array or object (undefined behavior, **guarded by
+    assertions**).
+    @post The JSON value remains unchanged.
+
+    @throw invalid_iterator.214 when called on `null` value
+
+    @liveexample{The following code shows an example for `front()`.,front}
+
+    @sa @ref back() -- access the last element
+
+    @since version 1.0.0
+    */
+    reference front()
+    {
+        return *begin();
+    }
+
+    /*!
+    @copydoc basic_json::front()
+    */
+    const_reference front() const
+    {
+        return *cbegin();
+    }
+
+    /*!
+    @brief access the last element
+
+    Returns a reference to the last element in the container. For a JSON
+    container `c`, the expression `c.back()` is equivalent to
+    @code {.cpp}
+    auto tmp = c.end();
+    --tmp;
+    return *tmp;
+    @endcode
+
+    @return In case of a structured type (array or object), a reference to the
+    last element is returned. In case of number, string, or boolean values, a
+    reference to the value is returned.
+
+    @complexity Constant.
+
+    @pre The JSON value must not be `null` (would throw `std::out_of_range`)
+    or an empty array or object (undefined behavior, **guarded by
+    assertions**).
+    @post The JSON value remains unchanged.
+
+    @throw invalid_iterator.214 when called on a `null` value. See example
+    below.
+
+    @liveexample{The following code shows an example for `back()`.,back}
+
+    @sa @ref front() -- access the first element
+
+    @since version 1.0.0
+    */
+    reference back()
+    {
+        auto tmp = end();
+        --tmp;
+        return *tmp;
+    }
+
+    /*!
+    @copydoc basic_json::back()
+    */
+    const_reference back() const
+    {
+        auto tmp = cend();
+        --tmp;
+        return *tmp;
+    }
+
+    /*!
+    @brief remove element given an iterator
+
+    Removes the element specified by iterator @a pos. The iterator @a pos must
+    be valid and dereferenceable. Thus the `end()` iterator (which is valid,
+    but is not dereferenceable) cannot be used as a value for @a pos.
+
+    If called on a primitive type other than `null`, the resulting JSON value
+    will be `null`.
+
+    @param[in] pos iterator to the element to remove
+    @return Iterator following the last removed element. If the iterator @a
+    pos refers to the last element, the `end()` iterator is returned.
+
+    @tparam IteratorType an @ref iterator or @ref const_iterator
+
+    @post Invalidates iterators and references at or after the point of the
+    erase, including the `end()` iterator.
+
+    @throw type_error.307 if called on a `null` value; example: `"cannot use
+    erase() with null"`
+    @throw invalid_iterator.202 if called on an iterator which does not belong
+    to the current JSON value; example: `"iterator does not fit current
+    value"`
+    @throw invalid_iterator.205 if called on a primitive type with invalid
+    iterator (i.e., any iterator which is not `begin()`); example: `"iterator
+    out of range"`
+
+    @complexity The complexity depends on the type:
+    - objects: amortized constant
+    - arrays: linear in distance between @a pos and the end of the container
+    - strings: linear in the length of the string
+    - other types: constant
+
+    @liveexample{The example shows the result of `erase()` for different JSON
+    types.,erase__IteratorType}
+
+    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
+    the given range
+    @sa @ref erase(const typename object_t::key_type&) -- removes the element
+    from an object at the given key
+    @sa @ref erase(const size_type) -- removes the element from an array at
+    the given index
+
+    @since version 1.0.0
+    */
+    template<class IteratorType, typename std::enable_if<
+                 std::is_same<IteratorType, typename basic_json_t::iterator>::value or
+                 std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type
+             = 0>
+    IteratorType erase(IteratorType pos)
+    {
+        // make sure iterator fits the current value
+        if (JSON_UNLIKELY(this != pos.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+        }
+
+        IteratorType result = end();
+
+        switch (m_type)
+        {
+            case value_t::boolean:
+            case value_t::number_float:
+            case value_t::number_integer:
+            case value_t::number_unsigned:
+            case value_t::string:
+            {
+                if (JSON_UNLIKELY(not pos.m_it.primitive_iterator.is_begin()))
+                {
+                    JSON_THROW(invalid_iterator::create(205, "iterator out of range"));
+                }
+
+                if (is_string())
+                {
+                    AllocatorType<string_t> alloc;
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);
+                    m_value.string = nullptr;
+                }
+
+                m_type = value_t::null;
+                assert_invariant();
+                break;
+            }
+
+            case value_t::object:
+            {
+                result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator);
+                break;
+            }
+
+            case value_t::array:
+            {
+                result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator);
+                break;
+            }
+
+            default:
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief remove elements given an iterator range
+
+    Removes the element specified by the range `[first; last)`. The iterator
+    @a first does not need to be dereferenceable if `first == last`: erasing
+    an empty range is a no-op.
+
+    If called on a primitive type other than `null`, the resulting JSON value
+    will be `null`.
+
+    @param[in] first iterator to the beginning of the range to remove
+    @param[in] last iterator past the end of the range to remove
+    @return Iterator following the last removed element. If the iterator @a
+    second refers to the last element, the `end()` iterator is returned.
+
+    @tparam IteratorType an @ref iterator or @ref const_iterator
+
+    @post Invalidates iterators and references at or after the point of the
+    erase, including the `end()` iterator.
+
+    @throw type_error.307 if called on a `null` value; example: `"cannot use
+    erase() with null"`
+    @throw invalid_iterator.203 if called on iterators which does not belong
+    to the current JSON value; example: `"iterators do not fit current value"`
+    @throw invalid_iterator.204 if called on a primitive type with invalid
+    iterators (i.e., if `first != begin()` and `last != end()`); example:
+    `"iterators out of range"`
+
+    @complexity The complexity depends on the type:
+    - objects: `log(size()) + std::distance(first, last)`
+    - arrays: linear in the distance between @a first and @a last, plus linear
+      in the distance between @a last and end of the container
+    - strings: linear in the length of the string
+    - other types: constant
+
+    @liveexample{The example shows the result of `erase()` for different JSON
+    types.,erase__IteratorType_IteratorType}
+
+    @sa @ref erase(IteratorType) -- removes the element at a given position
+    @sa @ref erase(const typename object_t::key_type&) -- removes the element
+    from an object at the given key
+    @sa @ref erase(const size_type) -- removes the element from an array at
+    the given index
+
+    @since version 1.0.0
+    */
+    template<class IteratorType, typename std::enable_if<
+                 std::is_same<IteratorType, typename basic_json_t::iterator>::value or
+                 std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type
+             = 0>
+    IteratorType erase(IteratorType first, IteratorType last)
+    {
+        // make sure iterator fits the current value
+        if (JSON_UNLIKELY(this != first.m_object or this != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value"));
+        }
+
+        IteratorType result = end();
+
+        switch (m_type)
+        {
+            case value_t::boolean:
+            case value_t::number_float:
+            case value_t::number_integer:
+            case value_t::number_unsigned:
+            case value_t::string:
+            {
+                if (JSON_LIKELY(not first.m_it.primitive_iterator.is_begin()
+                                or not last.m_it.primitive_iterator.is_end()))
+                {
+                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
+                }
+
+                if (is_string())
+                {
+                    AllocatorType<string_t> alloc;
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);
+                    m_value.string = nullptr;
+                }
+
+                m_type = value_t::null;
+                assert_invariant();
+                break;
+            }
+
+            case value_t::object:
+            {
+                result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator,
+                                              last.m_it.object_iterator);
+                break;
+            }
+
+            case value_t::array:
+            {
+                result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator,
+                                             last.m_it.array_iterator);
+                break;
+            }
+
+            default:
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief remove element from a JSON object given a key
+
+    Removes elements from a JSON object with the key value @a key.
+
+    @param[in] key value of the elements to remove
+
+    @return Number of elements removed. If @a ObjectType is the default
+    `std::map` type, the return value will always be `0` (@a key was not
+    found) or `1` (@a key was found).
+
+    @post References and iterators to the erased elements are invalidated.
+    Other references and iterators are not affected.
+
+    @throw type_error.307 when called on a type other than JSON object;
+    example: `"cannot use erase() with null"`
+
+    @complexity `log(size()) + count(key)`
+
+    @liveexample{The example shows the effect of `erase()`.,erase__key_type}
+
+    @sa @ref erase(IteratorType) -- removes the element at a given position
+    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
+    the given range
+    @sa @ref erase(const size_type) -- removes the element from an array at
+    the given index
+
+    @since version 1.0.0
+    */
+    size_type erase(const typename object_t::key_type& key)
+    {
+        // this erase only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            return m_value.object->erase(key);
+        }
+
+        JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief remove element from a JSON array given an index
+
+    Removes element from a JSON array at the index @a idx.
+
+    @param[in] idx index of the element to remove
+
+    @throw type_error.307 when called on a type other than JSON object;
+    example: `"cannot use erase() with null"`
+    @throw out_of_range.401 when `idx >= size()`; example: `"array index 17
+    is out of range"`
+
+    @complexity Linear in distance between @a idx and the end of the container.
+
+    @liveexample{The example shows the effect of `erase()`.,erase__size_type}
+
+    @sa @ref erase(IteratorType) -- removes the element at a given position
+    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
+    the given range
+    @sa @ref erase(const typename object_t::key_type&) -- removes the element
+    from an object at the given key
+
+    @since version 1.0.0
+    */
+    void erase(const size_type idx)
+    {
+        // this erase only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            if (JSON_UNLIKELY(idx >= size()))
+            {
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+            }
+
+            m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));
+        }
+        else
+        {
+            JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+        }
+    }
+
+    /// @}
+
+
+    ////////////
+    // lookup //
+    ////////////
+
+    /// @name lookup
+    /// @{
+
+    /*!
+    @brief find an element in a JSON object
+
+    Finds an element in a JSON object with key equivalent to @a key. If the
+    element is not found or the JSON value is not an object, end() is
+    returned.
+
+    @note This method always returns @ref end() when executed on a JSON type
+          that is not an object.
+
+    @param[in] key key value of the element to search for.
+
+    @return Iterator to an element with key equivalent to @a key. If no such
+    element is found or the JSON value is not an object, past-the-end (see
+    @ref end()) iterator is returned.
+
+    @complexity Logarithmic in the size of the JSON object.
+
+    @liveexample{The example shows how `find()` is used.,find__key_type}
+
+    @since version 1.0.0
+    */
+    template<typename KeyT>
+    iterator find(KeyT&& key)
+    {
+        auto result = end();
+
+        if (is_object())
+        {
+            result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief find an element in a JSON object
+    @copydoc find(KeyT&&)
+    */
+    template<typename KeyT>
+    const_iterator find(KeyT&& key) const
+    {
+        auto result = cend();
+
+        if (is_object())
+        {
+            result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief returns the number of occurrences of a key in a JSON object
+
+    Returns the number of elements with key @a key. If ObjectType is the
+    default `std::map` type, the return value will always be `0` (@a key was
+    not found) or `1` (@a key was found).
+
+    @note This method always returns `0` when executed on a JSON type that is
+          not an object.
+
+    @param[in] key key value of the element to count
+
+    @return Number of elements with key @a key. If the JSON value is not an
+    object, the return value will be `0`.
+
+    @complexity Logarithmic in the size of the JSON object.
+
+    @liveexample{The example shows how `count()` is used.,count}
+
+    @since version 1.0.0
+    */
+    template<typename KeyT>
+    size_type count(KeyT&& key) const
+    {
+        // return 0 for all nonobject types
+        return is_object() ? m_value.object->count(std::forward<KeyT>(key)) : 0;
+    }
+
+    /// @}
+
+
+    ///////////////
+    // iterators //
+    ///////////////
+
+    /// @name iterators
+    /// @{
+
+    /*!
+    @brief returns an iterator to the first element
+
+    Returns an iterator to the first element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return iterator to the first element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+
+    @liveexample{The following code shows an example for `begin()`.,begin}
+
+    @sa @ref cbegin() -- returns a const iterator to the beginning
+    @sa @ref end() -- returns an iterator to the end
+    @sa @ref cend() -- returns a const iterator to the end
+
+    @since version 1.0.0
+    */
+    iterator begin() noexcept
+    {
+        iterator result(this);
+        result.set_begin();
+        return result;
+    }
+
+    /*!
+    @copydoc basic_json::cbegin()
+    */
+    const_iterator begin() const noexcept
+    {
+        return cbegin();
+    }
+
+    /*!
+    @brief returns a const iterator to the first element
+
+    Returns a const iterator to the first element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return const iterator to the first element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).begin()`.
+
+    @liveexample{The following code shows an example for `cbegin()`.,cbegin}
+
+    @sa @ref begin() -- returns an iterator to the beginning
+    @sa @ref end() -- returns an iterator to the end
+    @sa @ref cend() -- returns a const iterator to the end
+
+    @since version 1.0.0
+    */
+    const_iterator cbegin() const noexcept
+    {
+        const_iterator result(this);
+        result.set_begin();
+        return result;
+    }
+
+    /*!
+    @brief returns an iterator to one past the last element
+
+    Returns an iterator to one past the last element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return iterator one past the last element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+
+    @liveexample{The following code shows an example for `end()`.,end}
+
+    @sa @ref cend() -- returns a const iterator to the end
+    @sa @ref begin() -- returns an iterator to the beginning
+    @sa @ref cbegin() -- returns a const iterator to the beginning
+
+    @since version 1.0.0
+    */
+    iterator end() noexcept
+    {
+        iterator result(this);
+        result.set_end();
+        return result;
+    }
+
+    /*!
+    @copydoc basic_json::cend()
+    */
+    const_iterator end() const noexcept
+    {
+        return cend();
+    }
+
+    /*!
+    @brief returns a const iterator to one past the last element
+
+    Returns a const iterator to one past the last element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return const iterator one past the last element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).end()`.
+
+    @liveexample{The following code shows an example for `cend()`.,cend}
+
+    @sa @ref end() -- returns an iterator to the end
+    @sa @ref begin() -- returns an iterator to the beginning
+    @sa @ref cbegin() -- returns a const iterator to the beginning
+
+    @since version 1.0.0
+    */
+    const_iterator cend() const noexcept
+    {
+        const_iterator result(this);
+        result.set_end();
+        return result;
+    }
+
+    /*!
+    @brief returns an iterator to the reverse-beginning
+
+    Returns an iterator to the reverse-beginning; that is, the last element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `reverse_iterator(end())`.
+
+    @liveexample{The following code shows an example for `rbegin()`.,rbegin}
+
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref crend() -- returns a const reverse iterator to the end
+
+    @since version 1.0.0
+    */
+    reverse_iterator rbegin() noexcept
+    {
+        return reverse_iterator(end());
+    }
+
+    /*!
+    @copydoc basic_json::crbegin()
+    */
+    const_reverse_iterator rbegin() const noexcept
+    {
+        return crbegin();
+    }
+
+    /*!
+    @brief returns an iterator to the reverse-end
+
+    Returns an iterator to the reverse-end; that is, one before the first
+    element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `reverse_iterator(begin())`.
+
+    @liveexample{The following code shows an example for `rend()`.,rend}
+
+    @sa @ref crend() -- returns a const reverse iterator to the end
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+
+    @since version 1.0.0
+    */
+    reverse_iterator rend() noexcept
+    {
+        return reverse_iterator(begin());
+    }
+
+    /*!
+    @copydoc basic_json::crend()
+    */
+    const_reverse_iterator rend() const noexcept
+    {
+        return crend();
+    }
+
+    /*!
+    @brief returns a const reverse iterator to the last element
+
+    Returns a const iterator to the reverse-beginning; that is, the last
+    element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`.
+
+    @liveexample{The following code shows an example for `crbegin()`.,crbegin}
+
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref crend() -- returns a const reverse iterator to the end
+
+    @since version 1.0.0
+    */
+    const_reverse_iterator crbegin() const noexcept
+    {
+        return const_reverse_iterator(cend());
+    }
+
+    /*!
+    @brief returns a const reverse iterator to one before the first
+
+    Returns a const reverse iterator to the reverse-end; that is, one before
+    the first element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).rend()`.
+
+    @liveexample{The following code shows an example for `crend()`.,crend}
+
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+
+    @since version 1.0.0
+    */
+    const_reverse_iterator crend() const noexcept
+    {
+        return const_reverse_iterator(cbegin());
+    }
+
+  public:
+    /*!
+    @brief wrapper to access iterator member functions in range-based for
+
+    This function allows to access @ref iterator::key() and @ref
+    iterator::value() during range-based for loops. In these loops, a
+    reference to the JSON values is returned, so there is no access to the
+    underlying iterator.
+
+    For loop without iterator_wrapper:
+
+    @code{cpp}
+    for (auto it = j_object.begin(); it != j_object.end(); ++it)
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    Range-based for loop without iterator proxy:
+
+    @code{cpp}
+    for (auto it : j_object)
+    {
+        // "it" is of type json::reference and has no key() member
+        std::cout << "value: " << it << '\n';
+    }
+    @endcode
+
+    Range-based for loop with iterator proxy:
+
+    @code{cpp}
+    for (auto it : json::iterator_wrapper(j_object))
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    @note When iterating over an array, `key()` will return the index of the
+          element as string (see example).
+
+    @param[in] ref  reference to a JSON value
+    @return iteration proxy object wrapping @a ref with an interface to use in
+            range-based for loops
+
+    @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @note The name of this function is not yet final and may change in the
+    future.
+
+    @deprecated This stream operator is deprecated and will be removed in
+                future 4.0.0 of the library. Please use @ref items() instead;
+                that is, replace `json::iterator_wrapper(j)` with `j.items()`.
+    */
+    JSON_DEPRECATED
+    static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept
+    {
+        return ref.items();
+    }
+
+    /*!
+    @copydoc iterator_wrapper(reference)
+    */
+    JSON_DEPRECATED
+    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept
+    {
+        return ref.items();
+    }
+
+    /*!
+    @brief helper to access iterator member functions in range-based for
+
+    This function allows to access @ref iterator::key() and @ref
+    iterator::value() during range-based for loops. In these loops, a
+    reference to the JSON values is returned, so there is no access to the
+    underlying iterator.
+
+    For loop without `items()` function:
+
+    @code{cpp}
+    for (auto it = j_object.begin(); it != j_object.end(); ++it)
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    Range-based for loop without `items()` function:
+
+    @code{cpp}
+    for (auto it : j_object)
+    {
+        // "it" is of type json::reference and has no key() member
+        std::cout << "value: " << it << '\n';
+    }
+    @endcode
+
+    Range-based for loop with `items()` function:
+
+    @code{cpp}
+    for (auto& el : j_object.items())
+    {
+        std::cout << "key: " << el.key() << ", value:" << el.value() << '\n';
+    }
+    @endcode
+
+    The `items()` function also allows to use
+    [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding)
+    (C++17):
+
+    @code{cpp}
+    for (auto& [key, val] : j_object.items())
+    {
+        std::cout << "key: " << key << ", value:" << val << '\n';
+    }
+    @endcode
+
+    @note When iterating over an array, `key()` will return the index of the
+          element as string (see example). For primitive types (e.g., numbers),
+          `key()` returns an empty string.
+
+    @return iteration proxy object wrapping @a ref with an interface to use in
+            range-based for loops
+
+    @liveexample{The following code shows how the function is used.,items}
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 3.1.0, structured bindings support since 3.5.0.
+    */
+    iteration_proxy<iterator> items() noexcept
+    {
+        return iteration_proxy<iterator>(*this);
+    }
+
+    /*!
+    @copydoc items()
+    */
+    iteration_proxy<const_iterator> items() const noexcept
+    {
+        return iteration_proxy<const_iterator>(*this);
+    }
+
+    /// @}
+
+
+    //////////////
+    // capacity //
+    //////////////
+
+    /// @name capacity
+    /// @{
+
+    /*!
+    @brief checks whether the container is empty.
+
+    Checks if a JSON value has no elements (i.e. whether its @ref size is `0`).
+
+    @return The return value depends on the different types and is
+            defined as follows:
+            Value type  | return value
+            ----------- | -------------
+            null        | `true`
+            boolean     | `false`
+            string      | `false`
+            number      | `false`
+            object      | result of function `object_t::empty()`
+            array       | result of function `array_t::empty()`
+
+    @liveexample{The following code uses `empty()` to check if a JSON
+    object contains any elements.,empty}
+
+    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
+    the Container concept; that is, their `empty()` functions have constant
+    complexity.
+
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @note This function does not return whether a string stored as JSON value
+    is empty - it returns whether the JSON container itself is empty which is
+    false in the case of a string.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `begin() == end()`.
+
+    @sa @ref size() -- returns the number of elements
+
+    @since version 1.0.0
+    */
+    bool empty() const noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::null:
+            {
+                // null values are empty
+                return true;
+            }
+
+            case value_t::array:
+            {
+                // delegate call to array_t::empty()
+                return m_value.array->empty();
+            }
+
+            case value_t::object:
+            {
+                // delegate call to object_t::empty()
+                return m_value.object->empty();
+            }
+
+            default:
+            {
+                // all other types are nonempty
+                return false;
+            }
+        }
+    }
+
+    /*!
+    @brief returns the number of elements
+
+    Returns the number of elements in a JSON value.
+
+    @return The return value depends on the different types and is
+            defined as follows:
+            Value type  | return value
+            ----------- | -------------
+            null        | `0`
+            boolean     | `1`
+            string      | `1`
+            number      | `1`
+            object      | result of function object_t::size()
+            array       | result of function array_t::size()
+
+    @liveexample{The following code calls `size()` on the different value
+    types.,size}
+
+    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
+    the Container concept; that is, their size() functions have constant
+    complexity.
+
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @note This function does not return the length of a string stored as JSON
+    value - it returns the number of elements in the JSON value which is 1 in
+    the case of a string.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `std::distance(begin(), end())`.
+
+    @sa @ref empty() -- checks whether the container is empty
+    @sa @ref max_size() -- returns the maximal number of elements
+
+    @since version 1.0.0
+    */
+    size_type size() const noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::null:
+            {
+                // null values are empty
+                return 0;
+            }
+
+            case value_t::array:
+            {
+                // delegate call to array_t::size()
+                return m_value.array->size();
+            }
+
+            case value_t::object:
+            {
+                // delegate call to object_t::size()
+                return m_value.object->size();
+            }
+
+            default:
+            {
+                // all other types have size 1
+                return 1;
+            }
+        }
+    }
+
+    /*!
+    @brief returns the maximum possible number of elements
+
+    Returns the maximum number of elements a JSON value is able to hold due to
+    system or library implementation limitations, i.e. `std::distance(begin(),
+    end())` for the JSON value.
+
+    @return The return value depends on the different types and is
+            defined as follows:
+            Value type  | return value
+            ----------- | -------------
+            null        | `0` (same as `size()`)
+            boolean     | `1` (same as `size()`)
+            string      | `1` (same as `size()`)
+            number      | `1` (same as `size()`)
+            object      | result of function `object_t::max_size()`
+            array       | result of function `array_t::max_size()`
+
+    @liveexample{The following code calls `max_size()` on the different value
+    types. Note the output is implementation specific.,max_size}
+
+    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
+    the Container concept; that is, their `max_size()` functions have constant
+    complexity.
+
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of returning `b.size()` where `b` is the largest
+      possible JSON value.
+
+    @sa @ref size() -- returns the number of elements
+
+    @since version 1.0.0
+    */
+    size_type max_size() const noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::array:
+            {
+                // delegate call to array_t::max_size()
+                return m_value.array->max_size();
+            }
+
+            case value_t::object:
+            {
+                // delegate call to object_t::max_size()
+                return m_value.object->max_size();
+            }
+
+            default:
+            {
+                // all other types have max_size() == size()
+                return size();
+            }
+        }
+    }
+
+    /// @}
+
+
+    ///////////////
+    // modifiers //
+    ///////////////
+
+    /// @name modifiers
+    /// @{
+
+    /*!
+    @brief clears the contents
+
+    Clears the content of a JSON value and resets it to the default value as
+    if @ref basic_json(value_t) would have been called with the current value
+    type from @ref type():
+
+    Value type  | initial value
+    ----------- | -------------
+    null        | `null`
+    boolean     | `false`
+    string      | `""`
+    number      | `0`
+    object      | `{}`
+    array       | `[]`
+
+    @post Has the same effect as calling
+    @code {.cpp}
+    *this = basic_json(type());
+    @endcode
+
+    @liveexample{The example below shows the effect of `clear()` to different
+    JSON types.,clear}
+
+    @complexity Linear in the size of the JSON value.
+
+    @iterators All iterators, pointers and references related to this container
+               are invalidated.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @sa @ref basic_json(value_t) -- constructor that creates an object with the
+        same value than calling `clear()`
+
+    @since version 1.0.0
+    */
+    void clear() noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::number_integer:
+            {
+                m_value.number_integer = 0;
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                m_value.number_unsigned = 0;
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                m_value.number_float = 0.0;
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                m_value.boolean = false;
+                break;
+            }
+
+            case value_t::string:
+            {
+                m_value.string->clear();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_value.array->clear();
+                break;
+            }
+
+            case value_t::object:
+            {
+                m_value.object->clear();
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    /*!
+    @brief add an object to an array
+
+    Appends the given element @a val to the end of the JSON value. If the
+    function is called on a JSON null value, an empty array is created before
+    appending @a val.
+
+    @param[in] val the value to add to the JSON array
+
+    @throw type_error.308 when called on a type other than JSON array or
+    null; example: `"cannot use push_back() with number"`
+
+    @complexity Amortized constant.
+
+    @liveexample{The example shows how `push_back()` and `+=` can be used to
+    add elements to a JSON array. Note how the `null` value was silently
+    converted to a JSON array.,push_back}
+
+    @since version 1.0.0
+    */
+    void push_back(basic_json&& val)
+    {
+        // push_back only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
+        {
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value = value_t::array;
+            assert_invariant();
+        }
+
+        // add element to array (move semantics)
+        m_value.array->push_back(std::move(val));
+        // invalidate object
+        val.m_type = value_t::null;
+    }
+
+    /*!
+    @brief add an object to an array
+    @copydoc push_back(basic_json&&)
+    */
+    reference operator+=(basic_json&& val)
+    {
+        push_back(std::move(val));
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an array
+    @copydoc push_back(basic_json&&)
+    */
+    void push_back(const basic_json& val)
+    {
+        // push_back only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
+        {
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value = value_t::array;
+            assert_invariant();
+        }
+
+        // add element to array
+        m_value.array->push_back(val);
+    }
+
+    /*!
+    @brief add an object to an array
+    @copydoc push_back(basic_json&&)
+    */
+    reference operator+=(const basic_json& val)
+    {
+        push_back(val);
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an object
+
+    Inserts the given element @a val to the JSON object. If the function is
+    called on a JSON null value, an empty object is created before inserting
+    @a val.
+
+    @param[in] val the value to add to the JSON object
+
+    @throw type_error.308 when called on a type other than JSON object or
+    null; example: `"cannot use push_back() with number"`
+
+    @complexity Logarithmic in the size of the container, O(log(`size()`)).
+
+    @liveexample{The example shows how `push_back()` and `+=` can be used to
+    add elements to a JSON object. Note how the `null` value was silently
+    converted to a JSON object.,push_back__object_t__value}
+
+    @since version 1.0.0
+    */
+    void push_back(const typename object_t::value_type& val)
+    {
+        // push_back only works for null objects or objects
+        if (JSON_UNLIKELY(not(is_null() or is_object())))
+        {
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value = value_t::object;
+            assert_invariant();
+        }
+
+        // add element to array
+        m_value.object->insert(val);
+    }
+
+    /*!
+    @brief add an object to an object
+    @copydoc push_back(const typename object_t::value_type&)
+    */
+    reference operator+=(const typename object_t::value_type& val)
+    {
+        push_back(val);
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an object
+
+    This function allows to use `push_back` with an initializer list. In case
+
+    1. the current value is an object,
+    2. the initializer list @a init contains only two elements, and
+    3. the first element of @a init is a string,
+
+    @a init is converted into an object element and added using
+    @ref push_back(const typename object_t::value_type&). Otherwise, @a init
+    is converted to a JSON value and added using @ref push_back(basic_json&&).
+
+    @param[in] init  an initializer list
+
+    @complexity Linear in the size of the initializer list @a init.
+
+    @note This function is required to resolve an ambiguous overload error,
+          because pairs like `{"key", "value"}` can be both interpreted as
+          `object_t::value_type` or `std::initializer_list<basic_json>`, see
+          https://github.com/nlohmann/json/issues/235 for more information.
+
+    @liveexample{The example shows how initializer lists are treated as
+    objects when possible.,push_back__initializer_list}
+    */
+    void push_back(initializer_list_t init)
+    {
+        if (is_object() and init.size() == 2 and (*init.begin())->is_string())
+        {
+            basic_json&& key = init.begin()->moved_or_copied();
+            push_back(typename object_t::value_type(
+                          std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied()));
+        }
+        else
+        {
+            push_back(basic_json(init));
+        }
+    }
+
+    /*!
+    @brief add an object to an object
+    @copydoc push_back(initializer_list_t)
+    */
+    reference operator+=(initializer_list_t init)
+    {
+        push_back(init);
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an array
+
+    Creates a JSON value from the passed parameters @a args to the end of the
+    JSON value. If the function is called on a JSON null value, an empty array
+    is created before appending the value created from @a args.
+
+    @param[in] args arguments to forward to a constructor of @ref basic_json
+    @tparam Args compatible types to create a @ref basic_json object
+
+    @throw type_error.311 when called on a type other than JSON array or
+    null; example: `"cannot use emplace_back() with number"`
+
+    @complexity Amortized constant.
+
+    @liveexample{The example shows how `push_back()` can be used to add
+    elements to a JSON array. Note how the `null` value was silently converted
+    to a JSON array.,emplace_back}
+
+    @since version 2.0.8
+    */
+    template<class... Args>
+    void emplace_back(Args&& ... args)
+    {
+        // emplace_back only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
+        {
+            JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value = value_t::array;
+            assert_invariant();
+        }
+
+        // add element to array (perfect forwarding)
+        m_value.array->emplace_back(std::forward<Args>(args)...);
+    }
+
+    /*!
+    @brief add an object to an object if key does not exist
+
+    Inserts a new element into a JSON object constructed in-place with the
+    given @a args if there is no element with the key in the container. If the
+    function is called on a JSON null value, an empty object is created before
+    appending the value created from @a args.
+
+    @param[in] args arguments to forward to a constructor of @ref basic_json
+    @tparam Args compatible types to create a @ref basic_json object
+
+    @return a pair consisting of an iterator to the inserted element, or the
+            already-existing element if no insertion happened, and a bool
+            denoting whether the insertion took place.
+
+    @throw type_error.311 when called on a type other than JSON object or
+    null; example: `"cannot use emplace() with number"`
+
+    @complexity Logarithmic in the size of the container, O(log(`size()`)).
+
+    @liveexample{The example shows how `emplace()` can be used to add elements
+    to a JSON object. Note how the `null` value was silently converted to a
+    JSON object. Further note how no value is added if there was already one
+    value stored with the same key.,emplace}
+
+    @since version 2.0.8
+    */
+    template<class... Args>
+    std::pair<iterator, bool> emplace(Args&& ... args)
+    {
+        // emplace only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_object())))
+        {
+            JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name())));
+        }
+
+        // transform null object into an object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value = value_t::object;
+            assert_invariant();
+        }
+
+        // add element to array (perfect forwarding)
+        auto res = m_value.object->emplace(std::forward<Args>(args)...);
+        // create result iterator and set iterator to the result of emplace
+        auto it = begin();
+        it.m_it.object_iterator = res.first;
+
+        // return pair of iterator and boolean
+        return {it, res.second};
+    }
+
+    /// Helper for insertion of an iterator
+    /// @note: This uses std::distance to support GCC 4.8,
+    ///        see https://github.com/nlohmann/json/pull/1257
+    template<typename... Args>
+    iterator insert_iterator(const_iterator pos, Args&& ... args)
+    {
+        iterator result(this);
+        assert(m_value.array != nullptr);
+
+        auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator);
+        m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);
+        result.m_it.array_iterator = m_value.array->begin() + insert_pos;
+
+        // This could have been written as:
+        // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
+        // but the return value of insert is missing in GCC 4.8, so it is written this way instead.
+
+        return result;
+    }
+
+    /*!
+    @brief inserts element
+
+    Inserts element @a val before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] val element to insert
+    @return iterator pointing to the inserted @a val.
+
+    @throw type_error.309 if called on JSON values other than arrays;
+    example: `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+
+    @complexity Constant plus linear in the distance between @a pos and end of
+    the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, const basic_json& val)
+    {
+        // insert only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            // check if iterator pos fits to this JSON value
+            if (JSON_UNLIKELY(pos.m_object != this))
+            {
+                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+            }
+
+            // insert to array and return iterator
+            return insert_iterator(pos, val);
+        }
+
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief inserts element
+    @copydoc insert(const_iterator, const basic_json&)
+    */
+    iterator insert(const_iterator pos, basic_json&& val)
+    {
+        return insert(pos, val);
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts @a cnt copies of @a val before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] cnt number of copies of @a val to insert
+    @param[in] val element to insert
+    @return iterator pointing to the first element inserted, or @a pos if
+    `cnt==0`
+
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+
+    @complexity Linear in @a cnt plus linear in the distance between @a pos
+    and end of the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert__count}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, size_type cnt, const basic_json& val)
+    {
+        // insert only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            // check if iterator pos fits to this JSON value
+            if (JSON_UNLIKELY(pos.m_object != this))
+            {
+                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+            }
+
+            // insert to array and return iterator
+            return insert_iterator(pos, cnt, val);
+        }
+
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts elements from range `[first, last)` before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+    @throw invalid_iterator.211 if @a first or @a last are iterators into
+    container for which insert is called; example: `"passed iterators may not
+    belong to container"`
+
+    @return iterator pointing to the first element inserted, or @a pos if
+    `first==last`
+
+    @complexity Linear in `std::distance(first, last)` plus linear in the
+    distance between @a pos and end of the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert__range}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, const_iterator first, const_iterator last)
+    {
+        // insert only works for arrays
+        if (JSON_UNLIKELY(not is_array()))
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+        }
+
+        // check if iterator pos fits to this JSON value
+        if (JSON_UNLIKELY(pos.m_object != this))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        if (JSON_UNLIKELY(first.m_object == this))
+        {
+            JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container"));
+        }
+
+        // insert to array and return iterator
+        return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts elements from initializer list @a ilist before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] ilist initializer list to insert the values from
+
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+
+    @return iterator pointing to the first element inserted, or @a pos if
+    `ilist` is empty
+
+    @complexity Linear in `ilist.size()` plus linear in the distance between
+    @a pos and end of the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert__ilist}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, initializer_list_t ilist)
+    {
+        // insert only works for arrays
+        if (JSON_UNLIKELY(not is_array()))
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+        }
+
+        // check if iterator pos fits to this JSON value
+        if (JSON_UNLIKELY(pos.m_object != this))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+        }
+
+        // insert to array and return iterator
+        return insert_iterator(pos, ilist.begin(), ilist.end());
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts elements from range `[first, last)`.
+
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.309 if called on JSON values other than objects; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    point to an object; example: `"iterators first and last must point to
+    objects"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+
+    @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number
+    of elements to insert.
+
+    @liveexample{The example shows how `insert()` is used.,insert__range_object}
+
+    @since version 3.0.0
+    */
+    void insert(const_iterator first, const_iterator last)
+    {
+        // insert only works for objects
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        // passed iterators must belong to objects
+        if (JSON_UNLIKELY(not first.m_object->is_object()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
+        }
+
+        m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);
+    }
+
+    /*!
+    @brief updates a JSON object from another object, overwriting existing keys
+
+    Inserts all values from JSON object @a j and overwrites existing keys.
+
+    @param[in] j  JSON object to read values from
+
+    @throw type_error.312 if called on JSON values other than objects; example:
+    `"cannot use update() with string"`
+
+    @complexity O(N*log(size() + N)), where N is the number of elements to
+                insert.
+
+    @liveexample{The example shows how `update()` is used.,update}
+
+    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
+
+    @since version 3.0.0
+    */
+    void update(const_reference j)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
+        }
+        if (JSON_UNLIKELY(not j.is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name())));
+        }
+
+        for (auto it = j.cbegin(); it != j.cend(); ++it)
+        {
+            m_value.object->operator[](it.key()) = it.value();
+        }
+    }
+
+    /*!
+    @brief updates a JSON object from another object, overwriting existing keys
+
+    Inserts all values from from range `[first, last)` and overwrites existing
+    keys.
+
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.312 if called on JSON values other than objects; example:
+    `"cannot use update() with string"`
+    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    point to an object; example: `"iterators first and last must point to
+    objects"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+
+    @complexity O(N*log(size() + N)), where N is the number of elements to
+                insert.
+
+    @liveexample{The example shows how `update()` is used__range.,update}
+
+    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
+
+    @since version 3.0.0
+    */
+    void update(const_iterator first, const_iterator last)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        // passed iterators must belong to objects
+        if (JSON_UNLIKELY(not first.m_object->is_object()
+                          or not last.m_object->is_object()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
+        }
+
+        for (auto it = first; it != last; ++it)
+        {
+            m_value.object->operator[](it.key()) = it.value();
+        }
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of the JSON value with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other JSON value to exchange the contents with
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how JSON values can be swapped with
+    `swap()`.,swap__reference}
+
+    @since version 1.0.0
+    */
+    void swap(reference other) noexcept (
+        std::is_nothrow_move_constructible<value_t>::value and
+        std::is_nothrow_move_assignable<value_t>::value and
+        std::is_nothrow_move_constructible<json_value>::value and
+        std::is_nothrow_move_assignable<json_value>::value
+    )
+    {
+        std::swap(m_type, other.m_type);
+        std::swap(m_value, other.m_value);
+        assert_invariant();
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of a JSON array with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other array to exchange the contents with
+
+    @throw type_error.310 when JSON value is not an array; example: `"cannot
+    use swap() with string"`
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how arrays can be swapped with
+    `swap()`.,swap__array_t}
+
+    @since version 1.0.0
+    */
+    void swap(array_t& other)
+    {
+        // swap only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            std::swap(*(m_value.array), other);
+        }
+        else
+        {
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of a JSON object with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other object to exchange the contents with
+
+    @throw type_error.310 when JSON value is not an object; example:
+    `"cannot use swap() with string"`
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how objects can be swapped with
+    `swap()`.,swap__object_t}
+
+    @since version 1.0.0
+    */
+    void swap(object_t& other)
+    {
+        // swap only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            std::swap(*(m_value.object), other);
+        }
+        else
+        {
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of a JSON string with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other string to exchange the contents with
+
+    @throw type_error.310 when JSON value is not a string; example: `"cannot
+    use swap() with boolean"`
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how strings can be swapped with
+    `swap()`.,swap__string_t}
+
+    @since version 1.0.0
+    */
+    void swap(string_t& other)
+    {
+        // swap only works for strings
+        if (JSON_LIKELY(is_string()))
+        {
+            std::swap(*(m_value.string), other);
+        }
+        else
+        {
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
+        }
+    }
+
+    /// @}
+
+  public:
+    //////////////////////////////////////////
+    // lexicographical comparison operators //
+    //////////////////////////////////////////
+
+    /// @name lexicographical comparison operators
+    /// @{
+
+    /*!
+    @brief comparison: equal
+
+    Compares two JSON values for equality according to the following rules:
+    - Two JSON values are equal if (1) they are from the same type and (2)
+      their stored values are the same according to their respective
+      `operator==`.
+    - Integer and floating-point numbers are automatically converted before
+      comparison. Note than two NaN values are always treated as unequal.
+    - Two JSON null values are equal.
+
+    @note Floating-point inside JSON values numbers are compared with
+    `json::number_float_t::operator==` which is `double::operator==` by
+    default. To compare floating-point while respecting an epsilon, an alternative
+    [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39)
+    could be used, for instance
+    @code {.cpp}
+    template<typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type>
+    inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept
+    {
+        return std::abs(a - b) <= epsilon;
+    }
+    @endcode
+
+    @note NaN values never compare equal to themselves or to other NaN values.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether the values @a lhs and @a rhs are equal
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @complexity Linear.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__equal}
+
+    @since version 1.0.0
+    */
+    friend bool operator==(const_reference lhs, const_reference rhs) noexcept
+    {
+        const auto lhs_type = lhs.type();
+        const auto rhs_type = rhs.type();
+
+        if (lhs_type == rhs_type)
+        {
+            switch (lhs_type)
+            {
+                case value_t::array:
+                    return (*lhs.m_value.array == *rhs.m_value.array);
+
+                case value_t::object:
+                    return (*lhs.m_value.object == *rhs.m_value.object);
+
+                case value_t::null:
+                    return true;
+
+                case value_t::string:
+                    return (*lhs.m_value.string == *rhs.m_value.string);
+
+                case value_t::boolean:
+                    return (lhs.m_value.boolean == rhs.m_value.boolean);
+
+                case value_t::number_integer:
+                    return (lhs.m_value.number_integer == rhs.m_value.number_integer);
+
+                case value_t::number_unsigned:
+                    return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned);
+
+                case value_t::number_float:
+                    return (lhs.m_value.number_float == rhs.m_value.number_float);
+
+                default:
+                    return false;
+            }
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
+        {
+            return (static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float);
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
+        {
+            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer));
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
+        {
+            return (static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float);
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
+        {
+            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned));
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
+        {
+            return (static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer);
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
+        {
+            return (lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned));
+        }
+
+        return false;
+    }
+
+    /*!
+    @brief comparison: equal
+    @copydoc operator==(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs == basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: equal
+    @copydoc operator==(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) == rhs);
+    }
+
+    /*!
+    @brief comparison: not equal
+
+    Compares two JSON values for inequality by calculating `not (lhs == rhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether the values @a lhs and @a rhs are not equal
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__notequal}
+
+    @since version 1.0.0
+    */
+    friend bool operator!=(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (lhs == rhs);
+    }
+
+    /*!
+    @brief comparison: not equal
+    @copydoc operator!=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs != basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: not equal
+    @copydoc operator!=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) != rhs);
+    }
+
+    /*!
+    @brief comparison: less than
+
+    Compares whether one JSON value @a lhs is less than another JSON value @a
+    rhs according to the following rules:
+    - If @a lhs and @a rhs have the same type, the values are compared using
+      the default `<` operator.
+    - Integer and floating-point numbers are automatically converted before
+      comparison
+    - In case @a lhs and @a rhs have different types, the values are ignored
+      and the order of the types is considered, see
+      @ref operator<(const value_t, const value_t).
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is less than @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__less}
+
+    @since version 1.0.0
+    */
+    friend bool operator<(const_reference lhs, const_reference rhs) noexcept
+    {
+        const auto lhs_type = lhs.type();
+        const auto rhs_type = rhs.type();
+
+        if (lhs_type == rhs_type)
+        {
+            switch (lhs_type)
+            {
+                case value_t::array:
+                    return (*lhs.m_value.array) < (*rhs.m_value.array);
+
+                case value_t::object:
+                    return *lhs.m_value.object < *rhs.m_value.object;
+
+                case value_t::null:
+                    return false;
+
+                case value_t::string:
+                    return *lhs.m_value.string < *rhs.m_value.string;
+
+                case value_t::boolean:
+                    return lhs.m_value.boolean < rhs.m_value.boolean;
+
+                case value_t::number_integer:
+                    return lhs.m_value.number_integer < rhs.m_value.number_integer;
+
+                case value_t::number_unsigned:
+                    return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned;
+
+                case value_t::number_float:
+                    return lhs.m_value.number_float < rhs.m_value.number_float;
+
+                default:
+                    return false;
+            }
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
+        {
+            return static_cast<number_float_t>(lhs.m_value.number_integer) < rhs.m_value.number_float;
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
+        {
+            return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_integer);
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
+        {
+            return static_cast<number_float_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_float;
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
+        {
+            return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_unsigned);
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
+        {
+            return lhs.m_value.number_integer < static_cast<number_integer_t>(rhs.m_value.number_unsigned);
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
+        {
+            return static_cast<number_integer_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_integer;
+        }
+
+        // We only reach this line if we cannot compare values. In that case,
+        // we compare types. Note we have to call the operator explicitly,
+        // because MSVC has problems otherwise.
+        return operator<(lhs_type, rhs_type);
+    }
+
+    /*!
+    @brief comparison: less than
+    @copydoc operator<(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs < basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: less than
+    @copydoc operator<(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) < rhs);
+    }
+
+    /*!
+    @brief comparison: less than or equal
+
+    Compares whether one JSON value @a lhs is less than or equal to another
+    JSON value by calculating `not (rhs < lhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is less than or equal to @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__greater}
+
+    @since version 1.0.0
+    */
+    friend bool operator<=(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (rhs < lhs);
+    }
+
+    /*!
+    @brief comparison: less than or equal
+    @copydoc operator<=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs <= basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: less than or equal
+    @copydoc operator<=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) <= rhs);
+    }
+
+    /*!
+    @brief comparison: greater than
+
+    Compares whether one JSON value @a lhs is greater than another
+    JSON value by calculating `not (lhs <= rhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is greater than to @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__lessequal}
+
+    @since version 1.0.0
+    */
+    friend bool operator>(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (lhs <= rhs);
+    }
+
+    /*!
+    @brief comparison: greater than
+    @copydoc operator>(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs > basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: greater than
+    @copydoc operator>(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) > rhs);
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+
+    Compares whether one JSON value @a lhs is greater than or equal to another
+    JSON value by calculating `not (lhs < rhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is greater than or equal to @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__greaterequal}
+
+    @since version 1.0.0
+    */
+    friend bool operator>=(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (lhs < rhs);
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+    @copydoc operator>=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs >= basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+    @copydoc operator>=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) >= rhs);
+    }
+
+    /// @}
+
+    ///////////////////
+    // serialization //
+    ///////////////////
+
+    /// @name serialization
+    /// @{
+
+    /*!
+    @brief serialize to stream
+
+    Serialize the given JSON value @a j to the output stream @a o. The JSON
+    value will be serialized using the @ref dump member function.
+
+    - The indentation of the output can be controlled with the member variable
+      `width` of the output stream @a o. For instance, using the manipulator
+      `std::setw(4)` on @a o sets the indentation level to `4` and the
+      serialization result is the same as calling `dump(4)`.
+
+    - The indentation character can be controlled with the member variable
+      `fill` of the output stream @a o. For instance, the manipulator
+      `std::setfill('\\t')` sets indentation to use a tab character rather than
+      the default space character.
+
+    @param[in,out] o  stream to serialize to
+    @param[in] j  JSON value to serialize
+
+    @return the stream @a o
+
+    @throw type_error.316 if a string stored inside the JSON value is not
+                          UTF-8 encoded
+
+    @complexity Linear.
+
+    @liveexample{The example below shows the serialization with different
+    parameters to `width` to adjust the indentation level.,operator_serialize}
+
+    @since version 1.0.0; indentation character added in version 3.0.0
+    */
+    friend std::ostream& operator<<(std::ostream& o, const basic_json& j)
+    {
+        // read width member and use it as indentation parameter if nonzero
+        const bool pretty_print = (o.width() > 0);
+        const auto indentation = (pretty_print ? o.width() : 0);
+
+        // reset width to 0 for subsequent calls to this stream
+        o.width(0);
+
+        // do the actual serialization
+        serializer s(detail::output_adapter<char>(o), o.fill());
+        s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));
+        return o;
+    }
+
+    /*!
+    @brief serialize to stream
+    @deprecated This stream operator is deprecated and will be removed in
+                future 4.0.0 of the library. Please use
+                @ref operator<<(std::ostream&, const basic_json&)
+                instead; that is, replace calls like `j >> o;` with `o << j;`.
+    @since version 1.0.0; deprecated since version 3.0.0
+    */
+    JSON_DEPRECATED
+    friend std::ostream& operator>>(const basic_json& j, std::ostream& o)
+    {
+        return o << j;
+    }
+
+    /// @}
+
+
+    /////////////////////
+    // deserialization //
+    /////////////////////
+
+    /// @name deserialization
+    /// @{
+
+    /*!
+    @brief deserialize from a compatible input
+
+    This function reads from a compatible input. Examples are:
+    - an array of 1-byte values
+    - strings with character/literal type with size of 1 byte
+    - input streams
+    - container with contiguous storage of 1-byte values. Compatible container
+      types include `std::vector`, `std::string`, `std::array`,
+      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
+      arrays can be used with `std::begin()`/`std::end()`. User-defined
+      containers can be used as long as they implement random-access iterators
+      and a contiguous storage.
+
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @pre The container storage is contiguous. Violating this precondition
+    yields undefined behavior. **This precondition is enforced with an
+    assertion.**
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with a noncompliant container and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @param[in] i  input to read from
+    @param[in] cb  a parser callback function of type @ref parser_callback_t
+    which is used to control the deserialization by filtering unwanted values
+    (optional)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return result of the deserialization
+
+    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
+    of input; expected string literal""`
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the parser callback function
+    @a cb has a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from an array.,parse__array__parser_callback_t}
+
+    @liveexample{The example below demonstrates the `parse()` function with
+    and without callback function.,parse__string__parser_callback_t}
+
+    @liveexample{The example below demonstrates the `parse()` function with
+    and without callback function.,parse__istream__parser_callback_t}
+
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from a contiguous container.,parse__contiguouscontainer__parser_callback_t}
+
+    @since version 2.0.3 (contiguous containers)
+    */
+    static basic_json parse(detail::input_adapter&& i,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
+    {
+        basic_json result;
+        parser(i, cb, allow_exceptions).parse(true, result);
+        return result;
+    }
+
+    static bool accept(detail::input_adapter&& i)
+    {
+        return parser(i).accept(true);
+    }
+
+    /*!
+    @brief generate SAX events
+
+    The SAX event lister must follow the interface of @ref json_sax.
+
+    This function reads from a compatible input. Examples are:
+    - an array of 1-byte values
+    - strings with character/literal type with size of 1 byte
+    - input streams
+    - container with contiguous storage of 1-byte values. Compatible container
+      types include `std::vector`, `std::string`, `std::array`,
+      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
+      arrays can be used with `std::begin()`/`std::end()`. User-defined
+      containers can be used as long as they implement random-access iterators
+      and a contiguous storage.
+
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @pre The container storage is contiguous. Violating this precondition
+    yields undefined behavior. **This precondition is enforced with an
+    assertion.**
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with a noncompliant container and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @param[in] i  input to read from
+    @param[in,out] sax  SAX event listener
+    @param[in] format  the format to parse (JSON, CBOR, MessagePack, or UBJSON)
+    @param[in] strict  whether the input has to be consumed completely
+
+    @return return value of the last processed SAX event
+
+    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
+    of input; expected string literal""`
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the SAX consumer @a sax has
+    a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `sax_parse()` function
+    reading from string and processing the events with a user-defined SAX
+    event consumer.,sax_parse}
+
+    @since version 3.2.0
+    */
+    template <typename SAX>
+    static bool sax_parse(detail::input_adapter&& i, SAX* sax,
+                          input_format_t format = input_format_t::json,
+                          const bool strict = true)
+    {
+        assert(sax);
+        switch (format)
+        {
+            case input_format_t::json:
+                return parser(std::move(i)).sax_parse(sax, strict);
+            default:
+                return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict);
+        }
+    }
+
+    /*!
+    @brief deserialize from an iterator range with contiguous storage
+
+    This function reads from an iterator range of a container with contiguous
+    storage of 1-byte values. Compatible container types include
+    `std::vector`, `std::string`, `std::array`, `std::valarray`, and
+    `std::initializer_list`. Furthermore, C-style arrays can be used with
+    `std::begin()`/`std::end()`. User-defined containers can be used as long
+    as they implement random-access iterators and a contiguous storage.
+
+    @pre The iterator range is contiguous. Violating this precondition yields
+    undefined behavior. **This precondition is enforced with an assertion.**
+    @pre Each element in the range has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with noncompliant iterators and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @tparam IteratorType iterator of container with contiguous storage
+    @param[in] first  begin of the range to parse (included)
+    @param[in] last  end of the range to parse (excluded)
+    @param[in] cb  a parser callback function of type @ref parser_callback_t
+    which is used to control the deserialization by filtering unwanted values
+    (optional)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return result of the deserialization
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the parser callback function
+    @a cb has a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from an iterator range.,parse__iteratortype__parser_callback_t}
+
+    @since version 2.0.3
+    */
+    template<class IteratorType, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static basic_json parse(IteratorType first, IteratorType last,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
+    {
+        basic_json result;
+        parser(detail::input_adapter(first, last), cb, allow_exceptions).parse(true, result);
+        return result;
+    }
+
+    template<class IteratorType, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static bool accept(IteratorType first, IteratorType last)
+    {
+        return parser(detail::input_adapter(first, last)).accept(true);
+    }
+
+    template<class IteratorType, class SAX, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static bool sax_parse(IteratorType first, IteratorType last, SAX* sax)
+    {
+        return parser(detail::input_adapter(first, last)).sax_parse(sax);
+    }
+
+    /*!
+    @brief deserialize from stream
+    @deprecated This stream operator is deprecated and will be removed in
+                version 4.0.0 of the library. Please use
+                @ref operator>>(std::istream&, basic_json&)
+                instead; that is, replace calls like `j << i;` with `i >> j;`.
+    @since version 1.0.0; deprecated since version 3.0.0
+    */
+    JSON_DEPRECATED
+    friend std::istream& operator<<(basic_json& j, std::istream& i)
+    {
+        return operator>>(i, j);
+    }
+
+    /*!
+    @brief deserialize from stream
+
+    Deserializes an input stream to a JSON value.
+
+    @param[in,out] i  input stream to read a serialized JSON value from
+    @param[in,out] j  JSON value to write the deserialized input to
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below shows how a JSON value is constructed by
+    reading a serialization from a stream.,operator_deserialize}
+
+    @sa parse(std::istream&, const parser_callback_t) for a variant with a
+    parser callback function to filter values while parsing
+
+    @since version 1.0.0
+    */
+    friend std::istream& operator>>(std::istream& i, basic_json& j)
+    {
+        parser(detail::input_adapter(i)).parse(false, j);
+        return i;
+    }
+
+    /// @}
+
+    ///////////////////////////
+    // convenience functions //
+    ///////////////////////////
+
+    /*!
+    @brief return the type as string
+
+    Returns the type name as string to be used in error messages - usually to
+    indicate that a function was called on a wrong JSON type.
+
+    @return a string representation of a the @a m_type member:
+            Value type  | return value
+            ----------- | -------------
+            null        | `"null"`
+            boolean     | `"boolean"`
+            string      | `"string"`
+            number      | `"number"` (for all number types)
+            object      | `"object"`
+            array       | `"array"`
+            discarded   | `"discarded"`
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @complexity Constant.
+
+    @liveexample{The following code exemplifies `type_name()` for all JSON
+    types.,type_name}
+
+    @sa @ref type() -- return the type of the JSON value
+    @sa @ref operator value_t() -- return the type of the JSON value (implicit)
+
+    @since version 1.0.0, public since 2.1.0, `const char*` and `noexcept`
+    since 3.0.0
+    */
+    const char* type_name() const noexcept
+    {
+        {
+            switch (m_type)
+            {
+                case value_t::null:
+                    return "null";
+                case value_t::object:
+                    return "object";
+                case value_t::array:
+                    return "array";
+                case value_t::string:
+                    return "string";
+                case value_t::boolean:
+                    return "boolean";
+                case value_t::discarded:
+                    return "discarded";
+                default:
+                    return "number";
+            }
+        }
+    }
+
+
+  private:
+    //////////////////////
+    // member variables //
+    //////////////////////
+
+    /// the type of the current element
+    value_t m_type = value_t::null;
+
+    /// the value of the current element
+    json_value m_value = {};
+
+    //////////////////////////////////////////
+    // binary serialization/deserialization //
+    //////////////////////////////////////////
+
+    /// @name binary serialization/deserialization support
+    /// @{
+
+  public:
+    /*!
+    @brief create a CBOR serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the CBOR (Concise
+    Binary Object Representation) serialization format. CBOR is a binary
+    serialization format which aims to be more compact than JSON itself, yet
+    more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    CBOR types according to the CBOR specification (RFC 7049):
+
+    JSON value type | value/range                                | CBOR type                          | first byte
+    --------------- | ------------------------------------------ | ---------------------------------- | ---------------
+    null            | `null`                                     | Null                               | 0xF6
+    boolean         | `true`                                     | True                               | 0xF5
+    boolean         | `false`                                    | False                              | 0xF4
+    number_integer  | -9223372036854775808..-2147483649          | Negative integer (8 bytes follow)  | 0x3B
+    number_integer  | -2147483648..-32769                        | Negative integer (4 bytes follow)  | 0x3A
+    number_integer  | -32768..-129                               | Negative integer (2 bytes follow)  | 0x39
+    number_integer  | -128..-25                                  | Negative integer (1 byte follow)   | 0x38
+    number_integer  | -24..-1                                    | Negative integer                   | 0x20..0x37
+    number_integer  | 0..23                                      | Integer                            | 0x00..0x17
+    number_integer  | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
+    number_integer  | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
+    number_integer  | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1A
+    number_integer  | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1B
+    number_unsigned | 0..23                                      | Integer                            | 0x00..0x17
+    number_unsigned | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
+    number_unsigned | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
+    number_unsigned | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1A
+    number_unsigned | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1B
+    number_float    | *any value*                                | Double-Precision Float             | 0xFB
+    string          | *length*: 0..23                            | UTF-8 string                       | 0x60..0x77
+    string          | *length*: 23..255                          | UTF-8 string (1 byte follow)       | 0x78
+    string          | *length*: 256..65535                       | UTF-8 string (2 bytes follow)      | 0x79
+    string          | *length*: 65536..4294967295                | UTF-8 string (4 bytes follow)      | 0x7A
+    string          | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow)      | 0x7B
+    array           | *size*: 0..23                              | array                              | 0x80..0x97
+    array           | *size*: 23..255                            | array (1 byte follow)              | 0x98
+    array           | *size*: 256..65535                         | array (2 bytes follow)             | 0x99
+    array           | *size*: 65536..4294967295                  | array (4 bytes follow)             | 0x9A
+    array           | *size*: 4294967296..18446744073709551615   | array (8 bytes follow)             | 0x9B
+    object          | *size*: 0..23                              | map                                | 0xA0..0xB7
+    object          | *size*: 23..255                            | map (1 byte follow)                | 0xB8
+    object          | *size*: 256..65535                         | map (2 bytes follow)               | 0xB9
+    object          | *size*: 65536..4294967295                  | map (4 bytes follow)               | 0xBA
+    object          | *size*: 4294967296..18446744073709551615   | map (8 bytes follow)               | 0xBB
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a CBOR value.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @note The following CBOR types are not used in the conversion:
+          - byte strings (0x40..0x5F)
+          - UTF-8 strings terminated by "break" (0x7F)
+          - arrays terminated by "break" (0x9F)
+          - maps terminated by "break" (0xBF)
+          - date/time (0xC0..0xC1)
+          - bignum (0xC2..0xC3)
+          - decimal fraction (0xC4)
+          - bigfloat (0xC5)
+          - tagged items (0xC6..0xD4, 0xD8..0xDB)
+          - expected conversions (0xD5..0xD7)
+          - simple values (0xE0..0xF3, 0xF8)
+          - undefined (0xF7)
+          - half and single-precision floats (0xF9-0xFA)
+          - break (0xFF)
+
+    @param[in] j  JSON value to serialize
+    @return MessagePack serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in CBOR format.,to_cbor}
+
+    @sa http://cbor.io
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        analogous deserialization
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
+
+    @since version 2.0.9
+    */
+    static std::vector<uint8_t> to_cbor(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_cbor(j, result);
+        return result;
+    }
+
+    static void to_cbor(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_cbor(j);
+    }
+
+    static void to_cbor(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_cbor(j);
+    }
+
+    /*!
+    @brief create a MessagePack serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the MessagePack
+    serialization format. MessagePack is a binary serialization format which
+    aims to be more compact than JSON itself, yet more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    MessagePack types according to the MessagePack specification:
+
+    JSON value type | value/range                       | MessagePack type | first byte
+    --------------- | --------------------------------- | ---------------- | ----------
+    null            | `null`                            | nil              | 0xC0
+    boolean         | `true`                            | true             | 0xC3
+    boolean         | `false`                           | false            | 0xC2
+    number_integer  | -9223372036854775808..-2147483649 | int64            | 0xD3
+    number_integer  | -2147483648..-32769               | int32            | 0xD2
+    number_integer  | -32768..-129                      | int16            | 0xD1
+    number_integer  | -128..-33                         | int8             | 0xD0
+    number_integer  | -32..-1                           | negative fixint  | 0xE0..0xFF
+    number_integer  | 0..127                            | positive fixint  | 0x00..0x7F
+    number_integer  | 128..255                          | uint 8           | 0xCC
+    number_integer  | 256..65535                        | uint 16          | 0xCD
+    number_integer  | 65536..4294967295                 | uint 32          | 0xCE
+    number_integer  | 4294967296..18446744073709551615  | uint 64          | 0xCF
+    number_unsigned | 0..127                            | positive fixint  | 0x00..0x7F
+    number_unsigned | 128..255                          | uint 8           | 0xCC
+    number_unsigned | 256..65535                        | uint 16          | 0xCD
+    number_unsigned | 65536..4294967295                 | uint 32          | 0xCE
+    number_unsigned | 4294967296..18446744073709551615  | uint 64          | 0xCF
+    number_float    | *any value*                       | float 64         | 0xCB
+    string          | *length*: 0..31                   | fixstr           | 0xA0..0xBF
+    string          | *length*: 32..255                 | str 8            | 0xD9
+    string          | *length*: 256..65535              | str 16           | 0xDA
+    string          | *length*: 65536..4294967295       | str 32           | 0xDB
+    array           | *size*: 0..15                     | fixarray         | 0x90..0x9F
+    array           | *size*: 16..65535                 | array 16         | 0xDC
+    array           | *size*: 65536..4294967295         | array 32         | 0xDD
+    object          | *size*: 0..15                     | fix map          | 0x80..0x8F
+    object          | *size*: 16..65535                 | map 16           | 0xDE
+    object          | *size*: 65536..4294967295         | map 32           | 0xDF
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a MessagePack value.
+
+    @note The following values can **not** be converted to a MessagePack value:
+          - strings with more than 4294967295 bytes
+          - arrays with more than 4294967295 elements
+          - objects with more than 4294967295 elements
+
+    @note The following MessagePack types are not used in the conversion:
+          - bin 8 - bin 32 (0xC4..0xC6)
+          - ext 8 - ext 32 (0xC7..0xC9)
+          - float 32 (0xCA)
+          - fixext 1 - fixext 16 (0xD4..0xD8)
+
+    @note Any MessagePack output created @ref to_msgpack can be successfully
+          parsed by @ref from_msgpack.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @param[in] j  JSON value to serialize
+    @return MessagePack serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in MessagePack format.,to_msgpack}
+
+    @sa http://msgpack.org
+    @sa @ref from_msgpack for the analogous deserialization
+    @sa @ref to_cbor(const basic_json& for the related CBOR format
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
+
+    @since version 2.0.9
+    */
+    static std::vector<uint8_t> to_msgpack(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_msgpack(j, result);
+        return result;
+    }
+
+    static void to_msgpack(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_msgpack(j);
+    }
+
+    static void to_msgpack(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_msgpack(j);
+    }
+
+    /*!
+    @brief create a UBJSON serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the UBJSON
+    (Universal Binary JSON) serialization format. UBJSON aims to be more compact
+    than JSON itself, yet more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    UBJSON types according to the UBJSON specification:
+
+    JSON value type | value/range                       | UBJSON type | marker
+    --------------- | --------------------------------- | ----------- | ------
+    null            | `null`                            | null        | `Z`
+    boolean         | `true`                            | true        | `T`
+    boolean         | `false`                           | false       | `F`
+    number_integer  | -9223372036854775808..-2147483649 | int64       | `L`
+    number_integer  | -2147483648..-32769               | int32       | `l`
+    number_integer  | -32768..-129                      | int16       | `I`
+    number_integer  | -128..127                         | int8        | `i`
+    number_integer  | 128..255                          | uint8       | `U`
+    number_integer  | 256..32767                        | int16       | `I`
+    number_integer  | 32768..2147483647                 | int32       | `l`
+    number_integer  | 2147483648..9223372036854775807   | int64       | `L`
+    number_unsigned | 0..127                            | int8        | `i`
+    number_unsigned | 128..255                          | uint8       | `U`
+    number_unsigned | 256..32767                        | int16       | `I`
+    number_unsigned | 32768..2147483647                 | int32       | `l`
+    number_unsigned | 2147483648..9223372036854775807   | int64       | `L`
+    number_float    | *any value*                       | float64     | `D`
+    string          | *with shortest length indicator*  | string      | `S`
+    array           | *see notes on optimized format*   | array       | `[`
+    object          | *see notes on optimized format*   | map         | `{`
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a UBJSON value.
+
+    @note The following values can **not** be converted to a UBJSON value:
+          - strings with more than 9223372036854775807 bytes (theoretical)
+          - unsigned integer numbers above 9223372036854775807
+
+    @note The following markers are not used in the conversion:
+          - `Z`: no-op values are not created.
+          - `C`: single-byte strings are serialized with `S` markers.
+
+    @note Any UBJSON output created @ref to_ubjson can be successfully parsed
+          by @ref from_ubjson.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @note The optimized formats for containers are supported: Parameter
+          @a use_size adds size information to the beginning of a container and
+          removes the closing marker. Parameter @a use_type further checks
+          whether all elements of a container have the same type and adds the
+          type marker to the beginning of the container. The @a use_type
+          parameter must only be used together with @a use_size = true. Note
+          that @a use_size = true alone may result in larger representations -
+          the benefit of this parameter is that the receiving side is
+          immediately informed on the number of elements of the container.
+
+    @param[in] j  JSON value to serialize
+    @param[in] use_size  whether to add size annotations to container types
+    @param[in] use_type  whether to add type annotations to container types
+                         (must be combined with @a use_size = true)
+    @return UBJSON serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in UBJSON format.,to_ubjson}
+
+    @sa http://ubjson.org
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        analogous deserialization
+    @sa @ref to_cbor(const basic_json& for the related CBOR format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+
+    @since version 3.1.0
+    */
+    static std::vector<uint8_t> to_ubjson(const basic_json& j,
+                                          const bool use_size = false,
+                                          const bool use_type = false)
+    {
+        std::vector<uint8_t> result;
+        to_ubjson(j, result, use_size, use_type);
+        return result;
+    }
+
+    static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o,
+                          const bool use_size = false, const bool use_type = false)
+    {
+        binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type);
+    }
+
+    static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,
+                          const bool use_size = false, const bool use_type = false)
+    {
+        binary_writer<char>(o).write_ubjson(j, use_size, use_type);
+    }
+
+
+    /*!
+    @brief Serializes the given JSON object `j` to BSON and returns a vector
+           containing the corresponding BSON-representation.
+
+    BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are
+    stored as a single entity (a so-called document).
+
+    The library uses the following mapping from JSON values types to BSON types:
+
+    JSON value type | value/range                       | BSON type   | marker
+    --------------- | --------------------------------- | ----------- | ------
+    null            | `null`                            | null        | 0x0A
+    boolean         | `true`, `false`                   | boolean     | 0x08
+    number_integer  | -9223372036854775808..-2147483649 | int64       | 0x12
+    number_integer  | -2147483648..2147483647           | int32       | 0x10
+    number_integer  | 2147483648..9223372036854775807   | int64       | 0x12
+    number_unsigned | 0..2147483647                     | int32       | 0x10
+    number_unsigned | 2147483648..9223372036854775807   | int64       | 0x12
+    number_unsigned | 9223372036854775808..18446744073709551615| --   | --
+    number_float    | *any value*                       | double      | 0x01
+    string          | *any value*                       | string      | 0x02
+    array           | *any value*                       | document    | 0x04
+    object          | *any value*                       | document    | 0x03
+
+    @warning The mapping is **incomplete**, since only JSON-objects (and things
+    contained therein) can be serialized to BSON.
+    Also, integers larger than 9223372036854775807 cannot be serialized to BSON,
+    and the keys may not contain U+0000, since they are serialized a
+    zero-terminated c-strings.
+
+    @throw out_of_range.407  if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807`
+    @throw out_of_range.409  if a key in `j` contains a NULL (U+0000)
+    @throw type_error.317    if `!j.is_object()`
+
+    @pre The input `j` is required to be an object: `j.is_object() == true`.
+
+    @note Any BSON output created via @ref to_bson can be successfully parsed
+          by @ref from_bson.
+
+    @param[in] j  JSON value to serialize
+    @return BSON serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in BSON format.,to_bson}
+
+    @sa http://bsonspec.org/spec.html
+    @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the
+        analogous deserialization
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
+    @sa @ref to_cbor(const basic_json&) for the related CBOR format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+    */
+    static std::vector<uint8_t> to_bson(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_bson(j, result);
+        return result;
+    }
+
+    /*!
+    @brief Serializes the given JSON object `j` to BSON and forwards the
+           corresponding BSON-representation to the given output_adapter `o`.
+    @param j The JSON object to convert to BSON.
+    @param o The output adapter that receives the binary BSON representation.
+    @pre The input `j` shall be an object: `j.is_object() == true`
+    @sa @ref to_bson(const basic_json&)
+    */
+    static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_bson(j);
+    }
+
+    /*!
+    @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>)
+    */
+    static void to_bson(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_bson(j);
+    }
+
+
+    /*!
+    @brief create a JSON value from an input in CBOR format
+
+    Deserializes a given input @a i to a JSON value using the CBOR (Concise
+    Binary Object Representation) serialization format.
+
+    The library maps CBOR types to JSON value types as follows:
+
+    CBOR type              | JSON value type | first byte
+    ---------------------- | --------------- | ----------
+    Integer                | number_unsigned | 0x00..0x17
+    Unsigned integer       | number_unsigned | 0x18
+    Unsigned integer       | number_unsigned | 0x19
+    Unsigned integer       | number_unsigned | 0x1A
+    Unsigned integer       | number_unsigned | 0x1B
+    Negative integer       | number_integer  | 0x20..0x37
+    Negative integer       | number_integer  | 0x38
+    Negative integer       | number_integer  | 0x39
+    Negative integer       | number_integer  | 0x3A
+    Negative integer       | number_integer  | 0x3B
+    Negative integer       | number_integer  | 0x40..0x57
+    UTF-8 string           | string          | 0x60..0x77
+    UTF-8 string           | string          | 0x78
+    UTF-8 string           | string          | 0x79
+    UTF-8 string           | string          | 0x7A
+    UTF-8 string           | string          | 0x7B
+    UTF-8 string           | string          | 0x7F
+    array                  | array           | 0x80..0x97
+    array                  | array           | 0x98
+    array                  | array           | 0x99
+    array                  | array           | 0x9A
+    array                  | array           | 0x9B
+    array                  | array           | 0x9F
+    map                    | object          | 0xA0..0xB7
+    map                    | object          | 0xB8
+    map                    | object          | 0xB9
+    map                    | object          | 0xBA
+    map                    | object          | 0xBB
+    map                    | object          | 0xBF
+    False                  | `false`         | 0xF4
+    True                   | `true`          | 0xF5
+    Null                   | `null`          | 0xF6
+    Half-Precision Float   | number_float    | 0xF9
+    Single-Precision Float | number_float    | 0xFA
+    Double-Precision Float | number_float    | 0xFB
+
+    @warning The mapping is **incomplete** in the sense that not all CBOR
+             types can be converted to a JSON value. The following CBOR types
+             are not supported and will yield parse errors (parse_error.112):
+             - byte strings (0x40..0x5F)
+             - date/time (0xC0..0xC1)
+             - bignum (0xC2..0xC3)
+             - decimal fraction (0xC4)
+             - bigfloat (0xC5)
+             - tagged items (0xC6..0xD4, 0xD8..0xDB)
+             - expected conversions (0xD5..0xD7)
+             - simple values (0xE0..0xF3, 0xF8)
+             - undefined (0xF7)
+
+    @warning CBOR allows map keys of any type, whereas JSON only allows
+             strings as keys in object values. Therefore, CBOR maps with keys
+             other than UTF-8 strings are rejected (parse_error.113).
+
+    @note Any CBOR output created @ref to_cbor can be successfully parsed by
+          @ref from_cbor.
+
+    @param[in] i  an input in CBOR format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported features from CBOR were
+    used in the given input @a v or if the input is not valid CBOR
+    @throw parse_error.113 if a string was expected as map key, but not found
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in CBOR
+    format to a JSON value.,from_cbor}
+
+    @sa http://cbor.io
+    @sa @ref to_cbor(const basic_json&) for the analogous serialization
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the
+        related MessagePack format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        related UBJSON format
+
+    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
+           consume input adapters, removed start_index parameter, and added
+           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
+           since 3.2.0
+    */
+    static basic_json from_cbor(detail::input_adapter&& i,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_cbor(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_cbor(A1 && a1, A2 && a2,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @brief create a JSON value from an input in MessagePack format
+
+    Deserializes a given input @a i to a JSON value using the MessagePack
+    serialization format.
+
+    The library maps MessagePack types to JSON value types as follows:
+
+    MessagePack type | JSON value type | first byte
+    ---------------- | --------------- | ----------
+    positive fixint  | number_unsigned | 0x00..0x7F
+    fixmap           | object          | 0x80..0x8F
+    fixarray         | array           | 0x90..0x9F
+    fixstr           | string          | 0xA0..0xBF
+    nil              | `null`          | 0xC0
+    false            | `false`         | 0xC2
+    true             | `true`          | 0xC3
+    float 32         | number_float    | 0xCA
+    float 64         | number_float    | 0xCB
+    uint 8           | number_unsigned | 0xCC
+    uint 16          | number_unsigned | 0xCD
+    uint 32          | number_unsigned | 0xCE
+    uint 64          | number_unsigned | 0xCF
+    int 8            | number_integer  | 0xD0
+    int 16           | number_integer  | 0xD1
+    int 32           | number_integer  | 0xD2
+    int 64           | number_integer  | 0xD3
+    str 8            | string          | 0xD9
+    str 16           | string          | 0xDA
+    str 32           | string          | 0xDB
+    array 16         | array           | 0xDC
+    array 32         | array           | 0xDD
+    map 16           | object          | 0xDE
+    map 32           | object          | 0xDF
+    negative fixint  | number_integer  | 0xE0-0xFF
+
+    @warning The mapping is **incomplete** in the sense that not all
+             MessagePack types can be converted to a JSON value. The following
+             MessagePack types are not supported and will yield parse errors:
+              - bin 8 - bin 32 (0xC4..0xC6)
+              - ext 8 - ext 32 (0xC7..0xC9)
+              - fixext 1 - fixext 16 (0xD4..0xD8)
+
+    @note Any MessagePack output created @ref to_msgpack can be successfully
+          parsed by @ref from_msgpack.
+
+    @param[in] i  an input in MessagePack format convertible to an input
+                  adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported features from MessagePack were
+    used in the given input @a i or if the input is not valid MessagePack
+    @throw parse_error.113 if a string was expected as map key, but not found
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    MessagePack format to a JSON value.,from_msgpack}
+
+    @sa http://msgpack.org
+    @sa @ref to_msgpack(const basic_json&) for the analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for
+        the related UBJSON format
+    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
+        the related BSON format
+
+    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
+           consume input adapters, removed start_index parameter, and added
+           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
+           since 3.2.0
+    */
+    static basic_json from_msgpack(detail::input_adapter&& i,
+                                   const bool strict = true,
+                                   const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_msgpack(A1 && a1, A2 && a2,
+                                   const bool strict = true,
+                                   const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @brief create a JSON value from an input in UBJSON format
+
+    Deserializes a given input @a i to a JSON value using the UBJSON (Universal
+    Binary JSON) serialization format.
+
+    The library maps UBJSON types to JSON value types as follows:
+
+    UBJSON type | JSON value type                         | marker
+    ----------- | --------------------------------------- | ------
+    no-op       | *no value, next value is read*          | `N`
+    null        | `null`                                  | `Z`
+    false       | `false`                                 | `F`
+    true        | `true`                                  | `T`
+    float32     | number_float                            | `d`
+    float64     | number_float                            | `D`
+    uint8       | number_unsigned                         | `U`
+    int8        | number_integer                          | `i`
+    int16       | number_integer                          | `I`
+    int32       | number_integer                          | `l`
+    int64       | number_integer                          | `L`
+    string      | string                                  | `S`
+    char        | string                                  | `C`
+    array       | array (optimized values are supported)  | `[`
+    object      | object (optimized values are supported) | `{`
+
+    @note The mapping is **complete** in the sense that any UBJSON value can
+          be converted to a JSON value.
+
+    @param[in] i  an input in UBJSON format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if a parse error occurs
+    @throw parse_error.113 if a string could not be parsed successfully
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    UBJSON format to a JSON value.,from_ubjson}
+
+    @sa http://ubjson.org
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
+        the related MessagePack format
+    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
+        the related BSON format
+
+    @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0
+    */
+    static basic_json from_ubjson(detail::input_adapter&& i,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_ubjson(A1 && a1, A2 && a2,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @brief Create a JSON value from an input in BSON format
+
+    Deserializes a given input @a i to a JSON value using the BSON (Binary JSON)
+    serialization format.
+
+    The library maps BSON record types to JSON value types as follows:
+
+    BSON type       | BSON marker byte | JSON value type
+    --------------- | ---------------- | ---------------------------
+    double          | 0x01             | number_float
+    string          | 0x02             | string
+    document        | 0x03             | object
+    array           | 0x04             | array
+    binary          | 0x05             | still unsupported
+    undefined       | 0x06             | still unsupported
+    ObjectId        | 0x07             | still unsupported
+    boolean         | 0x08             | boolean
+    UTC Date-Time   | 0x09             | still unsupported
+    null            | 0x0A             | null
+    Regular Expr.   | 0x0B             | still unsupported
+    DB Pointer      | 0x0C             | still unsupported
+    JavaScript Code | 0x0D             | still unsupported
+    Symbol          | 0x0E             | still unsupported
+    JavaScript Code | 0x0F             | still unsupported
+    int32           | 0x10             | number_integer
+    Timestamp       | 0x11             | still unsupported
+    128-bit decimal float | 0x13       | still unsupported
+    Max Key         | 0x7F             | still unsupported
+    Min Key         | 0xFF             | still unsupported
+
+    @warning The mapping is **incomplete**. The unsupported mappings
+             are indicated in the table above.
+
+    @param[in] i  an input in BSON format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.114 if an unsupported BSON record type is encountered
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    BSON format to a JSON value.,from_bson}
+
+    @sa http://bsonspec.org/spec.html
+    @sa @ref to_bson(const basic_json&) for the analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
+        the related MessagePack format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        related UBJSON format
+    */
+    static basic_json from_bson(detail::input_adapter&& i,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_bson(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_bson(A1 && a1, A2 && a2,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+
+
+    /// @}
+
+    //////////////////////////
+    // JSON Pointer support //
+    //////////////////////////
+
+    /// @name JSON Pointer functions
+    /// @{
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Uses a JSON pointer to retrieve a reference to the respective JSON value.
+    No bound checking is performed. Similar to @ref operator[](const typename
+    object_t::key_type&), `null` values are created in arrays and objects if
+    necessary.
+
+    In particular:
+    - If the JSON pointer points to an object key that does not exist, it
+      is created an filled with a `null` value before a reference to it
+      is returned.
+    - If the JSON pointer points to an array index that does not exist, it
+      is created an filled with a `null` value before a reference to it
+      is returned. All indices between the current maximum and the given
+      index are also filled with `null`.
+    - The special value `-` is treated as a synonym for the index past the
+      end.
+
+    @param[in] ptr  a JSON pointer
+
+    @return reference to the element pointed to by @a ptr
+
+    @complexity Constant.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+
+    @liveexample{The behavior is shown in the example.,operatorjson_pointer}
+
+    @since version 2.0.0
+    */
+    reference operator[](const json_pointer& ptr)
+    {
+        return ptr.get_unchecked(this);
+    }
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Uses a JSON pointer to retrieve a reference to the respective JSON value.
+    No bound checking is performed. The function does not change the JSON
+    value; no `null` values are created. In particular, the the special value
+    `-` yields an exception.
+
+    @param[in] ptr  JSON pointer to the desired element
+
+    @return const reference to the element pointed to by @a ptr
+
+    @complexity Constant.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+
+    @liveexample{The behavior is shown in the example.,operatorjson_pointer_const}
+
+    @since version 2.0.0
+    */
+    const_reference operator[](const json_pointer& ptr) const
+    {
+        return ptr.get_unchecked(this);
+    }
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Returns a reference to the element at with specified JSON pointer @a ptr,
+    with bounds checking.
+
+    @param[in] ptr  JSON pointer to the desired element
+
+    @return reference to the element pointed to by @a ptr
+
+    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
+    begins with '0'. See example below.
+
+    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
+    is not a number. See example below.
+
+    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
+    is out of range. See example below.
+
+    @throw out_of_range.402 if the array index '-' is used in the passed JSON
+    pointer @a ptr. As `at` provides checked access (and no elements are
+    implicitly inserted), the index '-' is always invalid. See example below.
+
+    @throw out_of_range.403 if the JSON pointer describes a key of an object
+    which cannot be found. See example below.
+
+    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
+    See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 2.0.0
+
+    @liveexample{The behavior is shown in the example.,at_json_pointer}
+    */
+    reference at(const json_pointer& ptr)
+    {
+        return ptr.get_checked(this);
+    }
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Returns a const reference to the element at with specified JSON pointer @a
+    ptr, with bounds checking.
+
+    @param[in] ptr  JSON pointer to the desired element
+
+    @return reference to the element pointed to by @a ptr
+
+    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
+    begins with '0'. See example below.
+
+    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
+    is not a number. See example below.
+
+    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
+    is out of range. See example below.
+
+    @throw out_of_range.402 if the array index '-' is used in the passed JSON
+    pointer @a ptr. As `at` provides checked access (and no elements are
+    implicitly inserted), the index '-' is always invalid. See example below.
+
+    @throw out_of_range.403 if the JSON pointer describes a key of an object
+    which cannot be found. See example below.
+
+    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
+    See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 2.0.0
+
+    @liveexample{The behavior is shown in the example.,at_json_pointer_const}
+    */
+    const_reference at(const json_pointer& ptr) const
+    {
+        return ptr.get_checked(this);
+    }
+
+    /*!
+    @brief return flattened JSON value
+
+    The function creates a JSON object whose keys are JSON pointers (see [RFC
+    6901](https://tools.ietf.org/html/rfc6901)) and whose values are all
+    primitive. The original JSON value can be restored using the @ref
+    unflatten() function.
+
+    @return an object that maps JSON pointers to primitive values
+
+    @note Empty objects and arrays are flattened to `null` and will not be
+          reconstructed correctly by the @ref unflatten() function.
+
+    @complexity Linear in the size the JSON value.
+
+    @liveexample{The following code shows how a JSON object is flattened to an
+    object whose keys consist of JSON pointers.,flatten}
+
+    @sa @ref unflatten() for the reverse function
+
+    @since version 2.0.0
+    */
+    basic_json flatten() const
+    {
+        basic_json result(value_t::object);
+        json_pointer::flatten("", *this, result);
+        return result;
+    }
+
+    /*!
+    @brief unflatten a previously flattened JSON value
+
+    The function restores the arbitrary nesting of a JSON value that has been
+    flattened before using the @ref flatten() function. The JSON value must
+    meet certain constraints:
+    1. The value must be an object.
+    2. The keys must be JSON pointers (see
+       [RFC 6901](https://tools.ietf.org/html/rfc6901))
+    3. The mapped values must be primitive JSON types.
+
+    @return the original JSON from a flattened version
+
+    @note Empty objects and arrays are flattened by @ref flatten() to `null`
+          values and can not unflattened to their original type. Apart from
+          this example, for a JSON value `j`, the following is always true:
+          `j == j.flatten().unflatten()`.
+
+    @complexity Linear in the size the JSON value.
+
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitive
+
+    @liveexample{The following code shows how a flattened JSON object is
+    unflattened into the original nested JSON object.,unflatten}
+
+    @sa @ref flatten() for the reverse function
+
+    @since version 2.0.0
+    */
+    basic_json unflatten() const
+    {
+        return json_pointer::unflatten(*this);
+    }
+
+    /// @}
+
+    //////////////////////////
+    // JSON Patch functions //
+    //////////////////////////
+
+    /// @name JSON Patch functions
+    /// @{
+
+    /*!
+    @brief applies a JSON patch
+
+    [JSON Patch](http://jsonpatch.com) defines a JSON document structure for
+    expressing a sequence of operations to apply to a JSON) document. With
+    this function, a JSON Patch is applied to the current JSON value by
+    executing all operations from the patch.
+
+    @param[in] json_patch  JSON patch document
+    @return patched document
+
+    @note The application of a patch is atomic: Either all operations succeed
+          and the patched document is returned or an exception is thrown. In
+          any case, the original value is not changed: the patch is applied
+          to a copy of the value.
+
+    @throw parse_error.104 if the JSON patch does not consist of an array of
+    objects
+
+    @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory
+    attributes are missing); example: `"operation add must have member path"`
+
+    @throw out_of_range.401 if an array index is out of range.
+
+    @throw out_of_range.403 if a JSON pointer inside the patch could not be
+    resolved successfully in the current JSON value; example: `"key baz not
+    found"`
+
+    @throw out_of_range.405 if JSON pointer has no parent ("add", "remove",
+    "move")
+
+    @throw other_error.501 if "test" operation was unsuccessful
+
+    @complexity Linear in the size of the JSON value and the length of the
+    JSON patch. As usually only a fraction of the JSON value is affected by
+    the patch, the complexity can usually be neglected.
+
+    @liveexample{The following code shows how a JSON patch is applied to a
+    value.,patch}
+
+    @sa @ref diff -- create a JSON patch by comparing two JSON values
+
+    @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
+    @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901)
+
+    @since version 2.0.0
+    */
+    basic_json patch(const basic_json& json_patch) const
+    {
+        // make a working copy to apply the patch to
+        basic_json result = *this;
+
+        // the valid JSON Patch operations
+        enum class patch_operations {add, remove, replace, move, copy, test, invalid};
+
+        const auto get_op = [](const std::string & op)
+        {
+            if (op == "add")
+            {
+                return patch_operations::add;
+            }
+            if (op == "remove")
+            {
+                return patch_operations::remove;
+            }
+            if (op == "replace")
+            {
+                return patch_operations::replace;
+            }
+            if (op == "move")
+            {
+                return patch_operations::move;
+            }
+            if (op == "copy")
+            {
+                return patch_operations::copy;
+            }
+            if (op == "test")
+            {
+                return patch_operations::test;
+            }
+
+            return patch_operations::invalid;
+        };
+
+        // wrapper for "add" operation; add value at ptr
+        const auto operation_add = [&result](json_pointer & ptr, basic_json val)
+        {
+            // adding to the root of the target document means replacing it
+            if (ptr.is_root())
+            {
+                result = val;
+            }
+            else
+            {
+                // make sure the top element of the pointer exists
+                json_pointer top_pointer = ptr.top();
+                if (top_pointer != ptr)
+                {
+                    result.at(top_pointer);
+                }
+
+                // get reference to parent of JSON pointer ptr
+                const auto last_path = ptr.pop_back();
+                basic_json& parent = result[ptr];
+
+                switch (parent.m_type)
+                {
+                    case value_t::null:
+                    case value_t::object:
+                    {
+                        // use operator[] to add value
+                        parent[last_path] = val;
+                        break;
+                    }
+
+                    case value_t::array:
+                    {
+                        if (last_path == "-")
+                        {
+                            // special case: append to back
+                            parent.push_back(val);
+                        }
+                        else
+                        {
+                            const auto idx = json_pointer::array_index(last_path);
+                            if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
+                            {
+                                // avoid undefined behavior
+                                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+                            }
+
+                            // default case: insert add offset
+                            parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
+                        }
+                        break;
+                    }
+
+                    // LCOV_EXCL_START
+                    default:
+                    {
+                        // if there exists a parent it cannot be primitive
+                        assert(false);
+                    }
+                        // LCOV_EXCL_STOP
+                }
+            }
+        };
+
+        // wrapper for "remove" operation; remove value at ptr
+        const auto operation_remove = [&result](json_pointer & ptr)
+        {
+            // get reference to parent of JSON pointer ptr
+            const auto last_path = ptr.pop_back();
+            basic_json& parent = result.at(ptr);
+
+            // remove child
+            if (parent.is_object())
+            {
+                // perform range check
+                auto it = parent.find(last_path);
+                if (JSON_LIKELY(it != parent.end()))
+                {
+                    parent.erase(it);
+                }
+                else
+                {
+                    JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found"));
+                }
+            }
+            else if (parent.is_array())
+            {
+                // note erase performs range check
+                parent.erase(static_cast<size_type>(json_pointer::array_index(last_path)));
+            }
+        };
+
+        // type check: top level value must be an array
+        if (JSON_UNLIKELY(not json_patch.is_array()))
+        {
+            JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
+        }
+
+        // iterate and apply the operations
+        for (const auto& val : json_patch)
+        {
+            // wrapper to get a value for an operation
+            const auto get_value = [&val](const std::string & op,
+                                          const std::string & member,
+                                          bool string_type) -> basic_json &
+            {
+                // find value
+                auto it = val.m_value.object->find(member);
+
+                // context-sensitive error message
+                const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'";
+
+                // check if desired value is present
+                if (JSON_UNLIKELY(it == val.m_value.object->end()))
+                {
+                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'"));
+                }
+
+                // check if result is of type string
+                if (JSON_UNLIKELY(string_type and not it->second.is_string()))
+                {
+                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'"));
+                }
+
+                // no error: return value
+                return it->second;
+            };
+
+            // type check: every element of the array must be an object
+            if (JSON_UNLIKELY(not val.is_object()))
+            {
+                JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
+            }
+
+            // collect mandatory members
+            const std::string op = get_value("op", "op", true);
+            const std::string path = get_value(op, "path", true);
+            json_pointer ptr(path);
+
+            switch (get_op(op))
+            {
+                case patch_operations::add:
+                {
+                    operation_add(ptr, get_value("add", "value", false));
+                    break;
+                }
+
+                case patch_operations::remove:
+                {
+                    operation_remove(ptr);
+                    break;
+                }
+
+                case patch_operations::replace:
+                {
+                    // the "path" location must exist - use at()
+                    result.at(ptr) = get_value("replace", "value", false);
+                    break;
+                }
+
+                case patch_operations::move:
+                {
+                    const std::string from_path = get_value("move", "from", true);
+                    json_pointer from_ptr(from_path);
+
+                    // the "from" location must exist - use at()
+                    basic_json v = result.at(from_ptr);
+
+                    // The move operation is functionally identical to a
+                    // "remove" operation on the "from" location, followed
+                    // immediately by an "add" operation at the target
+                    // location with the value that was just removed.
+                    operation_remove(from_ptr);
+                    operation_add(ptr, v);
+                    break;
+                }
+
+                case patch_operations::copy:
+                {
+                    const std::string from_path = get_value("copy", "from", true);
+                    const json_pointer from_ptr(from_path);
+
+                    // the "from" location must exist - use at()
+                    basic_json v = result.at(from_ptr);
+
+                    // The copy is functionally identical to an "add"
+                    // operation at the target location using the value
+                    // specified in the "from" member.
+                    operation_add(ptr, v);
+                    break;
+                }
+
+                case patch_operations::test:
+                {
+                    bool success = false;
+                    JSON_TRY
+                    {
+                        // check if "value" matches the one at "path"
+                        // the "path" location must exist - use at()
+                        success = (result.at(ptr) == get_value("test", "value", false));
+                    }
+                    JSON_INTERNAL_CATCH (out_of_range&)
+                    {
+                        // ignore out of range errors: success remains false
+                    }
+
+                    // throw an exception if test fails
+                    if (JSON_UNLIKELY(not success))
+                    {
+                        JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump()));
+                    }
+
+                    break;
+                }
+
+                case patch_operations::invalid:
+                {
+                    // op must be "add", "remove", "replace", "move", "copy", or
+                    // "test"
+                    JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid"));
+                }
+            }
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief creates a diff as a JSON patch
+
+    Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can
+    be changed into the value @a target by calling @ref patch function.
+
+    @invariant For two JSON values @a source and @a target, the following code
+    yields always `true`:
+    @code {.cpp}
+    source.patch(diff(source, target)) == target;
+    @endcode
+
+    @note Currently, only `remove`, `add`, and `replace` operations are
+          generated.
+
+    @param[in] source  JSON value to compare from
+    @param[in] target  JSON value to compare against
+    @param[in] path    helper value to create JSON pointers
+
+    @return a JSON patch to convert the @a source to @a target
+
+    @complexity Linear in the lengths of @a source and @a target.
+
+    @liveexample{The following code shows how a JSON patch is created as a
+    diff for two JSON values.,diff}
+
+    @sa @ref patch -- apply a JSON patch
+    @sa @ref merge_patch -- apply a JSON Merge Patch
+
+    @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
+
+    @since version 2.0.0
+    */
+    static basic_json diff(const basic_json& source, const basic_json& target,
+                           const std::string& path = "")
+    {
+        // the patch
+        basic_json result(value_t::array);
+
+        // if the values are the same, return empty patch
+        if (source == target)
+        {
+            return result;
+        }
+
+        if (source.type() != target.type())
+        {
+            // different types: replace value
+            result.push_back(
+            {
+                {"op", "replace"}, {"path", path}, {"value", target}
+            });
+        }
+        else
+        {
+            switch (source.type())
+            {
+                case value_t::array:
+                {
+                    // first pass: traverse common elements
+                    std::size_t i = 0;
+                    while (i < source.size() and i < target.size())
+                    {
+                        // recursive call to compare array values at index i
+                        auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i));
+                        result.insert(result.end(), temp_diff.begin(), temp_diff.end());
+                        ++i;
+                    }
+
+                    // i now reached the end of at least one array
+                    // in a second pass, traverse the remaining elements
+
+                    // remove my remaining elements
+                    const auto end_index = static_cast<difference_type>(result.size());
+                    while (i < source.size())
+                    {
+                        // add operations in reverse order to avoid invalid
+                        // indices
+                        result.insert(result.begin() + end_index, object(
+                        {
+                            {"op", "remove"},
+                            {"path", path + "/" + std::to_string(i)}
+                        }));
+                        ++i;
+                    }
+
+                    // add other remaining elements
+                    while (i < target.size())
+                    {
+                        result.push_back(
+                        {
+                            {"op", "add"},
+                            {"path", path + "/" + std::to_string(i)},
+                            {"value", target[i]}
+                        });
+                        ++i;
+                    }
+
+                    break;
+                }
+
+                case value_t::object:
+                {
+                    // first pass: traverse this object's elements
+                    for (auto it = source.cbegin(); it != source.cend(); ++it)
+                    {
+                        // escape the key name to be used in a JSON patch
+                        const auto key = json_pointer::escape(it.key());
+
+                        if (target.find(it.key()) != target.end())
+                        {
+                            // recursive call to compare object values at key it
+                            auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key);
+                            result.insert(result.end(), temp_diff.begin(), temp_diff.end());
+                        }
+                        else
+                        {
+                            // found a key that is not in o -> remove it
+                            result.push_back(object(
+                            {
+                                {"op", "remove"}, {"path", path + "/" + key}
+                            }));
+                        }
+                    }
+
+                    // second pass: traverse other object's elements
+                    for (auto it = target.cbegin(); it != target.cend(); ++it)
+                    {
+                        if (source.find(it.key()) == source.end())
+                        {
+                            // found a key that is not in this -> add it
+                            const auto key = json_pointer::escape(it.key());
+                            result.push_back(
+                            {
+                                {"op", "add"}, {"path", path + "/" + key},
+                                {"value", it.value()}
+                            });
+                        }
+                    }
+
+                    break;
+                }
+
+                default:
+                {
+                    // both primitive type: replace value
+                    result.push_back(
+                    {
+                        {"op", "replace"}, {"path", path}, {"value", target}
+                    });
+                    break;
+                }
+            }
+        }
+
+        return result;
+    }
+
+    /// @}
+
+    ////////////////////////////////
+    // JSON Merge Patch functions //
+    ////////////////////////////////
+
+    /// @name JSON Merge Patch functions
+    /// @{
+
+    /*!
+    @brief applies a JSON Merge Patch
+
+    The merge patch format is primarily intended for use with the HTTP PATCH
+    method as a means of describing a set of modifications to a target
+    resource's content. This function applies a merge patch to the current
+    JSON value.
+
+    The function implements the following algorithm from Section 2 of
+    [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396):
+
+    ```
+    define MergePatch(Target, Patch):
+      if Patch is an Object:
+        if Target is not an Object:
+          Target = {} // Ignore the contents and set it to an empty Object
+        for each Name/Value pair in Patch:
+          if Value is null:
+            if Name exists in Target:
+              remove the Name/Value pair from Target
+          else:
+            Target[Name] = MergePatch(Target[Name], Value)
+        return Target
+      else:
+        return Patch
+    ```
+
+    Thereby, `Target` is the current object; that is, the patch is applied to
+    the current value.
+
+    @param[in] apply_patch  the patch to apply
+
+    @complexity Linear in the lengths of @a patch.
+
+    @liveexample{The following code shows how a JSON Merge Patch is applied to
+    a JSON document.,merge_patch}
+
+    @sa @ref patch -- apply a JSON patch
+    @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396)
+
+    @since version 3.0.0
+    */
+    void merge_patch(const basic_json& apply_patch)
+    {
+        if (apply_patch.is_object())
+        {
+            if (not is_object())
+            {
+                *this = object();
+            }
+            for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it)
+            {
+                if (it.value().is_null())
+                {
+                    erase(it.key());
+                }
+                else
+                {
+                    operator[](it.key()).merge_patch(it.value());
+                }
+            }
+        }
+        else
+        {
+            *this = apply_patch;
+        }
+    }
+
+    /// @}
+};
+} // namespace nlohmann
+
+///////////////////////
+// nonmember support //
+///////////////////////
+
+// specialization of std::swap, and std::hash
+namespace std
+{
+
+/// hash value for JSON objects
+template<>
+struct hash<nlohmann::json>
+{
+    /*!
+    @brief return a hash value for a JSON object
+
+    @since version 1.0.0
+    */
+    std::size_t operator()(const nlohmann::json& j) const
+    {
+        // a naive hashing via the string representation
+        const auto& h = hash<nlohmann::json::string_t>();
+        return h(j.dump());
+    }
+};
+
+/// specialization for std::less<value_t>
+/// @note: do not remove the space after '<',
+///        see https://github.com/nlohmann/json/pull/679
+template<>
+struct less< ::nlohmann::detail::value_t>
+{
+    /*!
+    @brief compare two value_t enum values
+    @since version 3.0.0
+    */
+    bool operator()(nlohmann::detail::value_t lhs,
+                    nlohmann::detail::value_t rhs) const noexcept
+    {
+        return nlohmann::detail::operator<(lhs, rhs);
+    }
+};
+
+/*!
+@brief exchanges the values of two JSON objects
+
+@since version 1.0.0
+*/
+template<>
+inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept(
+    is_nothrow_move_constructible<nlohmann::json>::value and
+    is_nothrow_move_assignable<nlohmann::json>::value
+)
+{
+    j1.swap(j2);
+}
+
+} // namespace std
+
+/*!
+@brief user-defined string literal for JSON values
+
+This operator implements a user-defined string literal for JSON objects. It
+can be used by adding `"_json"` to a string literal and returns a JSON object
+if no parse error occurred.
+
+@param[in] s  a string representation of a JSON object
+@param[in] n  the length of string @a s
+@return a JSON object
+
+@since version 1.0.0
+*/
+inline nlohmann::json operator "" _json(const char* s, std::size_t n)
+{
+    return nlohmann::json::parse(s, s + n);
+}
+
+/*!
+@brief user-defined string literal for JSON pointer
+
+This operator implements a user-defined string literal for JSON Pointers. It
+can be used by adding `"_json_pointer"` to a string literal and returns a JSON pointer
+object if no parse error occurred.
+
+@param[in] s  a string representation of a JSON Pointer
+@param[in] n  the length of string @a s
+@return a JSON pointer object
+
+@since version 2.0.0
+*/
+inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)
+{
+    return nlohmann::json::json_pointer(std::string(s, n));
+}
+
+// #include <nlohmann/detail/macro_unscope.hpp>
+
+
+// restore GCC/clang diagnostic settings
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #pragma GCC diagnostic pop
+#endif
+#if defined(__clang__)
+    #pragma GCC diagnostic pop
+#endif
+
+// clean up
+#undef JSON_INTERNAL_CATCH
+#undef JSON_CATCH
+#undef JSON_THROW
+#undef JSON_TRY
+#undef JSON_LIKELY
+#undef JSON_UNLIKELY
+#undef JSON_DEPRECATED
+#undef JSON_HAS_CPP_14
+#undef JSON_HAS_CPP_17
+#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
+#undef NLOHMANN_BASIC_JSON_TPL
+
+
+#endif

From 4d9568ef60eb959af11864ab7c910f781194f170 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gohla?= <b.gohla@gmx.de>
Date: Mon, 5 Apr 2021 21:29:04 +0100
Subject: [PATCH 849/882] Revert "[prerequisites]: add JSON lib dependency"

This reverts commit 5926200db09ca4d0c5769edf24a3cf2e9f472d23.
---
 doc/manual/src/installation/prerequisites-source.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index 12758c5e1..6825af707 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -69,6 +69,3 @@
     `--disable-seccomp-sandboxing` option to the `configure` script (Not
     recommended unless your system doesn't support `libseccomp`). To get
     the library, visit <https://github.com/seccomp/libseccomp>.
- 
-  - Niels Lohmann's [JSON library](https://github.com/nlohmann/json).
-  

From 8a29052cb2f52ef2c82c36fb3818fd0f66349729 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Apr 2021 12:21:31 +0200
Subject: [PATCH 850/882] PathSubstitutionGoal: Clean up pipe

If there were many top-level goals (which are not destroyed until the
very end), commands like

  $ nix copy --to 'ssh://localhost?remote-store=/tmp/nix' \
    /run/current-system --no-check-sigs --substitute-on-destination

could fail with "Too many open files". So now we do some explicit
cleanup from amDone(). It would be cleaner to separate goals from
their temporary internal state, but that would be a bigger refactor.
---
 src/libstore/build/goal.cc              |  2 ++
 src/libstore/build/goal.hh              |  2 ++
 src/libstore/build/substitution-goal.cc | 31 +++++++++++++++++--------
 src/libstore/build/substitution-goal.hh |  4 +++-
 src/libstore/build/worker.cc            |  1 +
 src/libutil/util.cc                     | 17 ++++++++++----
 src/libutil/util.hh                     |  3 ++-
 7 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index 2dd7a4d37..9de40bdf2 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -78,6 +78,8 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
     }
     waiters.clear();
     worker.removeGoal(shared_from_this());
+
+    cleanup();
 }
 
 
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index fca4f2d00..e6bf628cb 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this<Goal>
     virtual string key() = 0;
 
     void amDone(ExitCode result, std::optional<Error> ex = {});
+
+    virtual void cleanup() { }
 };
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 7b1ac126e..e56cfadbe 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -20,15 +20,7 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker &
 
 PathSubstitutionGoal::~PathSubstitutionGoal()
 {
-    try {
-        if (thr.joinable()) {
-            // FIXME: signal worker thread to quit.
-            thr.join();
-            worker.childTerminated(this);
-        }
-    } catch (...) {
-        ignoreException();
-    }
+    cleanup();
 }
 
 
@@ -63,6 +55,8 @@ void PathSubstitutionGoal::tryNext()
 {
     trace("trying next substituter");
 
+    cleanup();
+
     if (subs.size() == 0) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
@@ -205,7 +199,7 @@ void PathSubstitutionGoal::tryToRun()
     thr = std::thread([this]() {
         try {
             /* Wake up the worker loop when we're done. */
-            Finally updateStats([this]() { outPipe.writeSide = -1; });
+            Finally updateStats([this]() { outPipe.writeSide.close(); });
 
             Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
             PushActivity pact(act.id);
@@ -288,4 +282,21 @@ void PathSubstitutionGoal::handleEOF(int fd)
     if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }
 
+
+void PathSubstitutionGoal::cleanup()
+{
+    try {
+        if (thr.joinable()) {
+            // FIXME: signal worker thread to quit.
+            thr.join();
+            worker.childTerminated(this);
+        }
+
+        outPipe.close();
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+
 }
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index 3b3cb7e32..70c806d23 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -14,7 +14,7 @@ struct PathSubstitutionGoal : public Goal
     StorePath storePath;
 
     /* The path the substituter refers to the path as. This will be
-     * different when the stores have different names. */
+       different when the stores have different names. */
     std::optional<StorePath> subPath;
 
     /* The remaining substituters. */
@@ -79,6 +79,8 @@ public:
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
     void handleEOF(int fd) override;
+
+    void cleanup() override;
 };
 
 }
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 6c04d3ed3..0f2ade348 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -128,6 +128,7 @@ void Worker::removeGoal(GoalPtr goal)
         nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
     else
         assert(false);
+
     if (topGoals.find(goal) != topGoals.end()) {
         topGoals.erase(goal);
         /* If a top-level goal failed, then kill all other goals
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index dea9c74b7..a6fcabbec 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -752,13 +752,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {}
 AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
 
 
-AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd}
+AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
 {
     that.fd = -1;
 }
 
 
-AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that)
+AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
 {
     close();
     fd = that.fd;
@@ -789,6 +789,7 @@ void AutoCloseFD::close()
         if (::close(fd) == -1)
             /* This should never happen. */
             throw SysError("closing file descriptor %1%", fd);
+        fd = -1;
     }
 }
 
@@ -822,6 +823,12 @@ void Pipe::create()
 }
 
 
+void Pipe::close()
+{
+    readSide.close();
+    writeSide.close();
+}
+
 
 //////////////////////////////////////////////////////////////////////
 
@@ -1121,7 +1128,7 @@ void runProgram2(const RunOptions & options)
         throw SysError("executing '%1%'", options.program);
     }, processOptions);
 
-    out.writeSide = -1;
+    out.writeSide.close();
 
     std::thread writerThread;
 
@@ -1134,7 +1141,7 @@ void runProgram2(const RunOptions & options)
 
 
     if (source) {
-        in.readSide = -1;
+        in.readSide.close();
         writerThread = std::thread([&]() {
             try {
                 std::vector<char> buf(8 * 1024);
@@ -1151,7 +1158,7 @@ void runProgram2(const RunOptions & options)
             } catch (...) {
                 promise.set_exception(std::current_exception());
             }
-            in.writeSide = -1;
+            in.writeSide.close();
         });
     }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index ad49c65b3..ef5e5012b 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -188,7 +188,6 @@ public:
 class AutoCloseFD
 {
     int fd;
-    void close();
 public:
     AutoCloseFD();
     AutoCloseFD(int fd);
@@ -200,6 +199,7 @@ public:
     int get() const;
     explicit operator bool() const;
     int release();
+    void close();
 };
 
 
@@ -216,6 +216,7 @@ class Pipe
 public:
     AutoCloseFD readSide, writeSide;
     void create();
+    void close();
 };
 
 

From 9b9e703df41d75949272059f9b8bc8b763e91fce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Apr 2021 13:10:02 +0200
Subject: [PATCH 851/882] restoreSignals() + restoreAffinity() ->
 restoreProcessContext()

---
 src/libmain/shared.cc      |  4 ++--
 src/libstore/ssh.cc        |  4 ++--
 src/libutil/util.cc        | 14 +++++++++++---
 src/libutil/util.hh        |  8 +++++---
 src/nix-build/nix-build.cc |  4 +---
 src/nix/develop.cc         |  3 +--
 src/nix/edit.cc            |  3 ++-
 src/nix/repl.cc            | 30 +++++++-----------------------
 src/nix/run.cc             |  4 +---
 9 files changed, 32 insertions(+), 42 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 09af57871..86930c2e3 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -310,7 +310,7 @@ void printVersion(const string & programName)
 
 void showManPage(const string & name)
 {
-    restoreSignals();
+    restoreProcessContext();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
     throw SysError("command 'man %1%' failed", name.c_str());
@@ -373,7 +373,7 @@ RunPager::RunPager()
             throw SysError("dupping stdin");
         if (!getenv("LESS"))
             setenv("LESS", "FRSXMK", 1);
-        restoreSignals();
+        restoreProcessContext();
         if (pager)
             execl("/bin/sh", "sh", "-c", pager, nullptr);
         execlp("pager", "pager", nullptr);
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 235eed37a..93f72675d 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -50,7 +50,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
     options.dieWithParent = false;
 
     conn->sshPid = startProcess([&]() {
-        restoreSignals();
+        restoreProcessContext();
 
         close(in.writeSide.get());
         close(out.readSide.get());
@@ -110,7 +110,7 @@ Path SSHMaster::startMaster()
     options.dieWithParent = false;
 
     state->sshMaster = startProcess([&]() {
-        restoreSignals();
+        restoreProcessContext();
 
         close(out.readSide.get());
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a6fcabbec..06821abb1 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1116,7 +1116,7 @@ void runProgram2(const RunOptions & options)
         Strings args_(options.args);
         args_.push_front(options.program);
 
-        restoreSignals();
+        restoreProcessContext();
 
         if (options.searchPath)
             execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
@@ -1612,12 +1612,19 @@ void startSignalHandlerThread()
     std::thread(signalHandlerThread, set).detach();
 }
 
-void restoreSignals()
+static void restoreSignals()
 {
     if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
         throw SysError("restoring signals");
 }
 
+void restoreProcessContext()
+{
+    restoreSignals();
+
+    restoreAffinity();
+}
+
 /* RAII helper to automatically deregister a callback. */
 struct InterruptCallbackImpl : InterruptCallback
 {
@@ -1680,10 +1687,11 @@ string showBytes(uint64_t bytes)
 }
 
 
+// FIXME: move to libstore/build
 void commonChildInit(Pipe & logPipe)
 {
     const static string pathNullDevice = "/dev/null";
-    restoreSignals();
+    restoreProcessContext();
 
     /* Put the child in a separate session (and thus a separate
        process group) so that it has no controlling terminal (meaning
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index ef5e5012b..4762d1e55 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -300,6 +300,11 @@ std::pair<int, std::string> runProgram(const RunOptions & options);
 void runProgram2(const RunOptions & options);
 
 
+/* Restore the original inherited Unix process context (such as signal
+   masks, stack size, CPU affinity). */
+void restoreProcessContext();
+
+
 class ExecError : public Error
 {
 public:
@@ -513,9 +518,6 @@ class Callback;
    on the current thread (and thus any threads created by it). */
 void startSignalHandlerThread();
 
-/* Restore default signal handling. */
-void restoreSignals();
-
 struct InterruptCallback
 {
     virtual ~InterruptCallback() { };
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index d46bc1f2b..9acbedda2 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -422,8 +422,6 @@ static void main_nix_build(int argc, char * * argv)
             } else
                 env[var.first] = var.second;
 
-        restoreAffinity();
-
         /* Run a shell using the derivation's environment.  For
            convenience, source $stdenv/setup to setup additional
            environment variables and shell functions.  Also don't
@@ -473,7 +471,7 @@ static void main_nix_build(int argc, char * * argv)
 
         auto argPtrs = stringsToCharPtrs(args);
 
-        restoreSignals();
+        restoreProcessContext();
 
         logger->stop();
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 7cc7b85be..498a7b45c 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -462,8 +462,7 @@ struct CmdDevelop : Common, MixEnvironment
         auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath}
             : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
 
-        restoreAffinity();
-        restoreSignals();
+        restoreProcessContext();
 
         execvp(shell.c_str(), stringsToCharPtrs(args).data());
 
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 6472dd27a..b26417b18 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -42,7 +42,8 @@ struct CmdEdit : InstallableCommand
 
         auto args = editorFor(pos);
 
-        restoreSignals();
+        restoreProcessContext();
+
         execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
         std::string command;
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index bce8d31dc..eed79c332 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -343,24 +343,6 @@ StringSet NixRepl::completePrefix(string prefix)
 }
 
 
-static int runProgram(const string & program, const Strings & args)
-{
-    Strings args2(args);
-    args2.push_front(program);
-
-    Pid pid;
-    pid = fork();
-    if (pid == -1) throw SysError("forking");
-    if (pid == 0) {
-        restoreAffinity();
-        execvp(program.c_str(), stringsToCharPtrs(args2).data());
-        _exit(1);
-    }
-
-    return pid.wait();
-}
-
-
 bool isVarName(const string & s)
 {
     if (s.size() == 0) return false;
@@ -462,7 +444,7 @@ bool NixRepl::processLine(string line)
         auto args = editorFor(pos);
         auto editor = args.front();
         args.pop_front();
-        runProgram(editor, args);
+        runProgram(editor, true, args);
 
         // Reload right after exiting the editor
         state->resetFileCache();
@@ -481,7 +463,7 @@ bool NixRepl::processLine(string line)
         state->callFunction(f, v, result, Pos());
 
         StorePath drvPath = getDerivationPath(result);
-        runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)});
+        runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
@@ -494,16 +476,18 @@ bool NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
+            try {
+                runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw});
                 auto drv = state->store->readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputsAndOptPaths(*state->store))
                     std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
+            } catch (ExecError &) {
             }
         } else if (command == ":i") {
-            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});
+            runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw});
         } else {
-            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw});
+            runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw});
         }
     }
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ba60e57d8..b5d8ab38a 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -31,9 +31,7 @@ struct RunCommon : virtual Command
     {
         stopProgressBar();
 
-        restoreSignals();
-
-        restoreAffinity();
+        restoreProcessContext();
 
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location

From 906adadacd2d1c98346a2f42c0b42a32d2806d94 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Apr 2021 13:40:13 +0200
Subject: [PATCH 852/882] Restore stack size in child processes

Fixes #4673.
---
 src/libutil/util.cc | 27 +++++++++++++++++++++++++++
 src/libutil/util.hh |  4 ++++
 src/nix/main.cc     | 13 +------------
 3 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 06821abb1..60b318559 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -32,6 +32,7 @@
 
 #ifdef __linux__
 #include <sys/prctl.h>
+#include <sys/resource.h>
 #endif
 
 
@@ -1618,11 +1619,37 @@ static void restoreSignals()
         throw SysError("restoring signals");
 }
 
+#if __linux__
+rlim_t savedStackSize = 0;
+#endif
+
+void setStackSize(size_t stackSize)
+{
+    #if __linux__
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+        savedStackSize = limit.rlim_cur;
+        limit.rlim_cur = stackSize;
+        setrlimit(RLIMIT_STACK, &limit);
+    }
+    #endif
+}
+
 void restoreProcessContext()
 {
     restoreSignals();
 
     restoreAffinity();
+
+    #if __linux__
+    if (savedStackSize) {
+        struct rlimit limit;
+        if (getrlimit(RLIMIT_STACK, &limit) == 0) {
+            limit.rlim_cur = savedStackSize;
+            setrlimit(RLIMIT_STACK, &limit);
+        }
+    }
+    #endif
 }
 
 /* RAII helper to automatically deregister a callback. */
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 4762d1e55..f84d0fb31 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -300,6 +300,10 @@ std::pair<int, std::string> runProgram(const RunOptions & options);
 void runProgram2(const RunOptions & options);
 
 
+/* Change the stack size. */
+void setStackSize(size_t stackSize);
+
+
 /* Restore the original inherited Unix process context (such as signal
    masks, stack size, CPU affinity). */
 void restoreProcessContext();
diff --git a/src/nix/main.cc b/src/nix/main.cc
index f8701ee56..008482be3 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -17,10 +17,6 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
-#if __linux__
-#include <sys/resource.h>
-#endif
-
 #include <nlohmann/json.hpp>
 
 extern std::string chrootHelperName;
@@ -335,14 +331,7 @@ int main(int argc, char * * argv)
 {
     // Increase the default stack size for the evaluator and for
     // libstdc++'s std::regex.
-    #if __linux__
-    rlim_t stackSize = 64 * 1024 * 1024;
-    struct rlimit limit;
-    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
-        limit.rlim_cur = stackSize;
-        setrlimit(RLIMIT_STACK, &limit);
-    }
-    #endif
+    nix::setStackSize(64 * 1024 * 1024);
 
     return nix::handleExceptions(argv[0], [&]() {
         nix::mainWrapped(argc, argv);

From 42f0246698cff491f084547f7aaa9e778f629065 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Apr 2021 13:51:57 +0200
Subject: [PATCH 853/882] Revert "libfetchers/tarball: Lock on effectiveUrl"

This reverts commit fc6bfb261d50102016ed812ecf9949d41fe539f7.

Fixes #4672.
---
 src/libfetchers/fetchers.hh |  8 +-------
 src/libfetchers/github.cc   |  6 +++---
 src/libfetchers/tarball.cc  | 19 +++++--------------
 3 files changed, 9 insertions(+), 24 deletions(-)

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index c6b219c02..a72cfafa4 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -145,13 +145,7 @@ DownloadFileResult downloadFile(
     bool immutable,
     const Headers & headers = {});
 
-struct DownloadTarballMeta
-{
-    time_t lastModified;
-    std::string effectiveUrl;
-};
-
-std::pair<Tree, DownloadTarballMeta> downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 3e5ad75a8..8352ef02d 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified));
+        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", uint64_t(meta.lastModified)}
+                {"lastModified", uint64_t(lastModified)}
             },
             tree.storePath,
             true);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index bd05bb2f1..b8d7d2c70 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -109,7 +109,7 @@ DownloadFileResult downloadFile(
     };
 }
 
-std::pair<Tree, DownloadTarballMeta> downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -127,10 +127,7 @@ std::pair<Tree, DownloadTarballMeta> downloadTarball(
     if (cached && !cached->expired)
         return {
             Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
-            {
-                .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")),
-                .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url),
-            },
+            getIntAttr(cached->infoAttrs, "lastModified")
         };
 
     auto res = downloadFile(store, url, name, immutable, headers);
@@ -155,7 +152,6 @@ std::pair<Tree, DownloadTarballMeta> downloadTarball(
 
     Attrs infoAttrs({
         {"lastModified", uint64_t(lastModified)},
-        {"effectiveUrl", res.effectiveUrl},
         {"etag", res.etag},
     });
 
@@ -168,10 +164,7 @@ std::pair<Tree, DownloadTarballMeta> downloadTarball(
 
     return {
         Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
-        {
-            .lastModified = lastModified,
-            .effectiveUrl = res.effectiveUrl,
-        },
+        lastModified,
     };
 }
 
@@ -230,11 +223,9 @@ struct TarballInputScheme : InputScheme
         return true;
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        Input input(_input);
-        auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false);
-        input.attrs.insert_or_assign("url", meta.effectiveUrl);
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
         return {std::move(tree), input};
     }
 };

From c2dfda007e777a33cd487a5631f0d143f915dd43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 9 Apr 2021 23:13:08 +0200
Subject: [PATCH 854/882] add tests for zstd compression

---
 src/libstore/binary-cache-store.cc |  3 +++
 tests/local.mk                     |  1 +
 tests/zstd.sh                      | 28 ++++++++++++++++++++++++++++
 3 files changed, 32 insertions(+)
 create mode 100644 tests/zstd.sh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4f5f8607d..09e1c254b 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -179,6 +179,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
+           compression == "zstd" ? ".zst" :
+           compression == "lzip" ? ".lzip" :
+           compression == "lz4" ? ".lz4" :
            compression == "br" ? ".br" :
            "");
 
diff --git a/tests/local.mk b/tests/local.mk
index aa8b4f9bf..82894af78 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -27,6 +27,7 @@ nix_tests = \
   signing.sh \
   shell.sh \
   brotli.sh \
+  zstd.sh \
   pure-eval.sh \
   check.sh \
   plugins.sh \
diff --git a/tests/zstd.sh b/tests/zstd.sh
new file mode 100644
index 000000000..ba7c20501
--- /dev/null
+++ b/tests/zstd.sh
@@ -0,0 +1,28 @@
+source common.sh
+
+clearStore
+clearCache
+
+cacheURI="file://$cacheDir?compression=zstd"
+
+outPath=$(nix-build dependencies.nix --no-out-link)
+
+nix copy --to $cacheURI $outPath
+
+HASH=$(nix hash path $outPath)
+
+clearStore
+clearCacheCache
+
+nix copy --from $cacheURI $outPath --no-check-sigs
+
+if ls $cacheDir/nar/*.zst &> /dev/null; then
+    echo "files do exist"
+else
+    echo "nars do not exist"
+    exit 1
+fi
+
+HASH2=$(nix hash path $outPath)
+
+[[ $HASH = $HASH2 ]]

From 35faff73259521fdb0bf884db969c4774fced739 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Apr 2021 05:21:11 +0000
Subject: [PATCH 855/882] Bump cachix/cachix-action from v9 to v10

Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from v9 to v10.
- [Release notes](https://github.com/cachix/cachix-action/releases)
- [Commits](https://github.com/cachix/cachix-action/compare/v9...73e75d1a0cd4330597a571e8f9dedb41faa2fc4e)

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 33035ca1e..17a79dc97 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,7 +15,7 @@ jobs:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v13
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v9
+    - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -45,7 +45,7 @@ jobs:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@v13
-    - uses: cachix/cachix-action@v9
+    - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'

From d04969ffa53ce5f62efbfe02a85cfc12ec6097d6 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@floxdev.com>
Date: Mon, 12 Apr 2021 12:36:14 -0400
Subject: [PATCH 856/882] EvalSettings::getDefaultNixPath: respect
 {restrict,pure}Eval

Otherwise Nix may look to invalid locations for channels.
---
 src/libexpr/eval.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 3afe2e47b..936bccc8c 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2091,9 +2091,12 @@ Strings EvalSettings::getDefaultNixPath()
         }
     };
 
-    add(getHome() + "/.nix-defexpr/channels");
-    add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
-    add(settings.nixStateDir + "/profiles/per-user/root/channels");
+    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
+        add(getHome() + "/.nix-defexpr/channels");
+        add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
+        add(settings.nixStateDir + "/profiles/per-user/root/channels");
+    }
+
     return res;
 }
 

From ed29610cc60f28822845dbe9d1cae01baca3dd6a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 13 Apr 2021 11:25:28 +0200
Subject: [PATCH 857/882] Fix registerDrvOutput with the daemon

Resolve a protocol issue that caused the daemon to endlessly wait for
some information that the client doesn't ever send
---
 src/libstore/daemon.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index affd60472..0be9d2c54 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -887,7 +887,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputId = DrvOutput::parse(readString(from));
         auto outputPath = StorePath(readString(from));
-        auto resolvedDrv = StorePath(readString(from));
         store->registerDrvOutput(Realisation{
             .id = outputId, .outPath = outputPath});
         logger->stopWork();

From 3550a32b25068aa8b1670ef7fd2af2b821f35dc1 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 8 Jan 2021 22:09:35 +0100
Subject: [PATCH 858/882] primops/derivation: use position of currently
 evaluated attribute

* The position of the `name`-attribute appears in the trace.
* If e.g. `meta` has no `outPath`-attribute, a `cannot coerce set to
  string` error will be thrown where `pos` points to `name =` which is
  highly misleading.
---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 428adf4c2..f9eefddd0 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -953,7 +953,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     }
 
                 } else {
-                    auto s = state.coerceToString(posDrvName, *i->value, context, true);
+                    auto s = state.coerceToString(*i->pos, *i->value, context, true);
                     drv.env.emplace(key, s);
                     if (i->name == state.sBuilder) drv.builder = s;
                     else if (i->name == state.sSystem) drv.platform = s;

From 7c76964daa0d1ca07fd609f5eb28b51afd1246b7 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 8 Jan 2021 22:27:00 +0100
Subject: [PATCH 859/882] libexpr: misc improvements for proper error position

When working on some more complex Nix code, there are sometimes rather
unhelpful or misleading error messages, especially if coerce-errors are
thrown.

This patch is a first steps towards improving that. I'm happy to file
more changes after that, but I'd like to gather some feedback first.

To summarize, this patch does the following things:

* Attrsets (a.k.a. `Bindings` in `libexpr`) now have a `Pos`. This is
  helpful e.g. to identify which attribute-set in `listToAttrs` is
  invalid.

* The `Value`-struct has a new method named `determinePos` which tries
  to guess the position of a value and falls back to a default if that's
  not possible.

  This can be used to provide better messages if a coercion fails.

* The new `determinePos`-API is used by `builtins.concatMap` now. With
  that change, Nix shows the exact position in the error where a wrong
  value was returned by the lambda.

  To make sure it's still obvious that `concatMap` is the problem,
  another stack-frame was added.

* The changes described above can be added to every other `primop`, but
  first I'd like to get some feedback about the overall approach.
---
 src/libexpr/attr-set.hh |  1 +
 src/libexpr/eval.cc     | 11 +++++++++++
 src/libexpr/nixexpr.hh  |  4 +++-
 src/libexpr/parser.y    |  2 +-
 src/libexpr/primops.cc  | 31 +++++++++++++++++++++----------
 src/libexpr/value.hh    |  2 ++
 6 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 6d68e5df3..1da8d91df 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -35,6 +35,7 @@ class Bindings
 {
 public:
     typedef uint32_t size_t;
+    Pos *pos;
 
 private:
     size_t size_, capacity_;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 3afe2e47b..7fd44c2cf 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -201,6 +201,15 @@ string showType(const Value & v)
     }
 }
 
+Pos Value::determinePos(const Pos &pos) const
+{
+    switch (internalType) {
+        case tAttrs: return *attrs->pos;
+        case tLambda: return lambda.fun->pos;
+        case tApp: return app.left->determinePos(pos);
+        default: return pos;
+    }
+}
 
 bool Value::isTrivial() const
 {
@@ -1060,6 +1069,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos));
         v.attrs->sort(); // FIXME: inefficient
     }
+
+    v.attrs->pos = &pos;
 }
 
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 8df8055b3..51a14cd59 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -180,6 +180,7 @@ struct ExprOpHasAttr : Expr
 struct ExprAttrs : Expr
 {
     bool recursive;
+    Pos pos;
     struct AttrDef {
         bool inherited;
         Expr * e;
@@ -199,7 +200,8 @@ struct ExprAttrs : Expr
     };
     typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
     DynamicAttrDefs dynamicAttrs;
-    ExprAttrs() : recursive(false) { };
+    ExprAttrs(const Pos &pos) : recursive(false), pos(pos) { };
+    ExprAttrs() : recursive(false), pos(noPos) { };
     COMMON_METHODS
 };
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 49d995bb9..f948dde47 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -478,7 +478,7 @@ binds
           $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
       }
     }
-  | { $$ = new ExprAttrs; }
+  | { $$ = new ExprAttrs(makeCurPos(@0, data)); }
   ;
 
 attrs
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index f9eefddd0..c7bbfc050 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2149,21 +2149,27 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         state.forceAttrs(v2, pos);
 
         Bindings::iterator j = v2.attrs->find(state.sName);
-        if (j == v2.attrs->end())
-            throw TypeError({
-                .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
-                .errPos = pos
+        if (j == v2.attrs->end()) {
+            auto e = TypeError({
+                .msg = hintfmt("'name' attribute missing for 'listToAttrs'"),
+                .errPos = *v2.attrs->pos
             });
-        string name = state.forceStringNoCtx(*j->value, pos);
+            e.addTrace(pos, hintfmt("while invoking '%s'", "listToAttrs"));
+            throw e;
+        }
+        string name = state.forceStringNoCtx(*j->value, *j->pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
-            if (j2 == v2.attrs->end())
-                throw TypeError({
-                    .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
-                    .errPos = pos
+            if (j2 == v2.attrs->end()) {
+                auto e = TypeError({
+                    .msg = hintfmt("'value' attribute missing for 'listToAttrs'"),
+                    .errPos = *v2.attrs->pos
                 });
+                e.addTrace(pos, hintfmt("while invoking '%s'", "listToAttrs"));
+                throw e;
+            }
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }
@@ -2804,7 +2810,12 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
-        state.forceList(lists[n], pos);
+        try {
+            state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)));
+        } catch (TypeError &e) {
+            e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap"));
+            throw e;
+        }
         len += lists[n].listSize();
     }
 
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index b317c1898..a1f131f9e 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -341,6 +341,8 @@ public:
         return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size;
     }
 
+    Pos determinePos(const Pos &pos) const;
+
     /* Check whether forcing this value requires a trivial amount of
        computation. In particular, function applications are
        non-trivial. */

From f60473a07752fa409cec2c3ed818af99327a571c Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 11 Apr 2021 12:14:05 +0200
Subject: [PATCH 860/882] libexpr/primops: Move attr name extraction into its
 own function

This now takes care of providing positioning for both the faulting value
and the faulting function call in case of an error.
---
 src/libexpr/primops.cc | 86 ++++++++++++++++++++++++++----------------
 1 file changed, 54 insertions(+), 32 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c7bbfc050..7f3c298ba 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -547,18 +547,42 @@ typedef list<Value *> ValueList;
 #endif
 
 
+static Bindings::iterator extractAttrNameFromPrimopCall(
+        EvalState &state,
+        string funcName,
+        string attrName,
+        Bindings *attrSet,
+        const Pos &pos
+) {
+    Bindings::iterator value = attrSet->find(state.symbols.create(attrName));
+    if (value == attrSet->end()) {
+        auto e = TypeError({
+            .msg = hintfmt("attribute '%s' missing for call to '%s'", attrName, funcName),
+            .errPos = *attrSet->pos,
+        });
+
+        // Adding another trace for the function name to make it clear
+        // which call received wrong arguments.
+        e.addTrace(pos, hintfmt("while invoking '%s'", funcName));
+        throw e;
+    }
+
+    return value;
+}
+
 static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], pos);
 
     /* Get the start set. */
-    Bindings::iterator startSet =
-        args[0]->attrs->find(state.symbols.create("startSet"));
-    if (startSet == args[0]->attrs->end())
-        throw EvalError({
-            .msg = hintfmt("attribute 'startSet' required"),
-            .errPos = pos
-        });
+    Bindings::iterator startSet = extractAttrNameFromPrimopCall(
+        state,
+        "genericClosure",
+        "startSet",
+        args[0]->attrs,
+        pos
+    );
+
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -566,13 +590,14 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         workSet.push_back(startSet->value->listElems()[n]);
 
     /* Get the operator. */
-    Bindings::iterator op =
-        args[0]->attrs->find(state.symbols.create("operator"));
-    if (op == args[0]->attrs->end())
-        throw EvalError({
-            .msg = hintfmt("attribute 'operator' required"),
-            .errPos = pos
-        });
+    Bindings::iterator op = extractAttrNameFromPrimopCall(
+        state,
+        "genericClosure",
+        "operator",
+        args[0]->attrs,
+        pos
+    );
+
     state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
@@ -2148,28 +2173,25 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         Value & v2(*args[0]->listElems()[i]);
         state.forceAttrs(v2, pos);
 
-        Bindings::iterator j = v2.attrs->find(state.sName);
-        if (j == v2.attrs->end()) {
-            auto e = TypeError({
-                .msg = hintfmt("'name' attribute missing for 'listToAttrs'"),
-                .errPos = *v2.attrs->pos
-            });
-            e.addTrace(pos, hintfmt("while invoking '%s'", "listToAttrs"));
-            throw e;
-        }
+        Bindings::iterator j = extractAttrNameFromPrimopCall(
+            state,
+            "listToAttrs",
+            state.sName,
+            v2.attrs,
+            pos
+        );
+
         string name = state.forceStringNoCtx(*j->value, *j->pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
-            Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
-            if (j2 == v2.attrs->end()) {
-                auto e = TypeError({
-                    .msg = hintfmt("'value' attribute missing for 'listToAttrs'"),
-                    .errPos = *v2.attrs->pos
-                });
-                e.addTrace(pos, hintfmt("while invoking '%s'", "listToAttrs"));
-                throw e;
-            }
+            Bindings::iterator j2 = extractAttrNameFromPrimopCall(
+                state,
+                "listToAttrs",
+                state.sValue,
+                v2.attrs,
+                pos
+            );
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }

From ad7b47dc85c4c5fc01543bcd237bd050b50d9f4e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 11 Apr 2021 13:55:07 +0200
Subject: [PATCH 861/882] primops/libexpr: use new attr-call extractor
 everywhere; use function's pos if attr-set pos == noPos

---
 src/libexpr/primops.cc | 70 ++++++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 26 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7f3c298ba..e03f811b8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -556,15 +556,29 @@ static Bindings::iterator extractAttrNameFromPrimopCall(
 ) {
     Bindings::iterator value = attrSet->find(state.symbols.create(attrName));
     if (value == attrSet->end()) {
-        auto e = TypeError({
-            .msg = hintfmt("attribute '%s' missing for call to '%s'", attrName, funcName),
-            .errPos = *attrSet->pos,
-        });
+        hintformat errorMsg = hintfmt(
+            "attribute '%s' missing for call to '%s'",
+            attrName,
+            funcName
+        );
 
-        // Adding another trace for the function name to make it clear
-        // which call received wrong arguments.
-        e.addTrace(pos, hintfmt("while invoking '%s'", funcName));
-        throw e;
+        Pos aPos = *attrSet->pos;
+        if (aPos == noPos) {
+            throw TypeError({
+                .msg = errorMsg,
+                .errPos = pos,
+            });
+        } else {
+            auto e = TypeError({
+                .msg = errorMsg,
+                .errPos = aPos,
+            });
+
+            // Adding another trace for the function name to make it clear
+            // which call received wrong arguments.
+            e.addTrace(pos, hintfmt("while invoking '%s'", funcName));
+            throw e;
+        }
     }
 
     return value;
@@ -841,12 +855,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     state.forceAttrs(*args[0], pos);
 
     /* Figure out the name first (for stack backtraces). */
-    Bindings::iterator attr = args[0]->attrs->find(state.sName);
-    if (attr == args[0]->attrs->end())
-        throw EvalError({
-            .msg = hintfmt("required attribute 'name' missing"),
-            .errPos = pos
-        });
+    Bindings::iterator attr = extractAttrNameFromPrimopCall(
+        state,
+        "derivationStrict",
+        state.sName,
+        args[0]->attrs,
+        pos
+    );
+
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
@@ -1394,12 +1410,13 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         if (i != v2.attrs->end())
             prefix = state.forceStringNoCtx(*i->value, pos);
 
-        i = v2.attrs->find(state.symbols.create("path"));
-        if (i == v2.attrs->end())
-            throw EvalError({
-                .msg = hintfmt("attribute 'path' missing"),
-                .errPos = pos
-            });
+        i = extractAttrNameFromPrimopCall(
+            state,
+            "findFile",
+            "path",
+            v2.attrs,
+            pos
+        );
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -2041,12 +2058,13 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     string attr = state.forceStringNoCtx(*args[0], pos);
     state.forceAttrs(*args[1], pos);
     // !!! Should we create a symbol here or just do a lookup?
-    Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
-    if (i == args[1]->attrs->end())
-        throw EvalError({
-            .msg = hintfmt("attribute '%1%' missing", attr),
-            .errPos = pos
-        });
+    Bindings::iterator i = extractAttrNameFromPrimopCall(
+        state,
+        "getAttr",
+        attr,
+        args[1]->attrs,
+        pos
+    );
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);

From 864ef0e93dfd58b147d46c340305acf2f9c6b314 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 13 Apr 2021 23:08:59 +0200
Subject: [PATCH 862/882] libexpr/primops: review

---
 src/libexpr/primops.cc | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e03f811b8..ff2f302ed 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -547,13 +547,13 @@ typedef list<Value *> ValueList;
 #endif
 
 
-static Bindings::iterator extractAttrNameFromPrimopCall(
-        EvalState &state,
-        string funcName,
-        string attrName,
-        Bindings *attrSet,
-        const Pos &pos
-) {
+static Bindings::iterator getAttr(
+    EvalState & state,
+    string funcName,
+    string attrName,
+    Bindings * attrSet,
+    const Pos & pos)
+{
     Bindings::iterator value = attrSet->find(state.symbols.create(attrName));
     if (value == attrSet->end()) {
         hintformat errorMsg = hintfmt(
@@ -589,7 +589,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     state.forceAttrs(*args[0], pos);
 
     /* Get the start set. */
-    Bindings::iterator startSet = extractAttrNameFromPrimopCall(
+    Bindings::iterator startSet = getAttr(
         state,
         "genericClosure",
         "startSet",
@@ -604,7 +604,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         workSet.push_back(startSet->value->listElems()[n]);
 
     /* Get the operator. */
-    Bindings::iterator op = extractAttrNameFromPrimopCall(
+    Bindings::iterator op = getAttr(
         state,
         "genericClosure",
         "operator",
@@ -855,7 +855,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     state.forceAttrs(*args[0], pos);
 
     /* Figure out the name first (for stack backtraces). */
-    Bindings::iterator attr = extractAttrNameFromPrimopCall(
+    Bindings::iterator attr = getAttr(
         state,
         "derivationStrict",
         state.sName,
@@ -1410,7 +1410,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         if (i != v2.attrs->end())
             prefix = state.forceStringNoCtx(*i->value, pos);
 
-        i = extractAttrNameFromPrimopCall(
+        i = getAttr(
             state,
             "findFile",
             "path",
@@ -2058,7 +2058,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     string attr = state.forceStringNoCtx(*args[0], pos);
     state.forceAttrs(*args[1], pos);
     // !!! Should we create a symbol here or just do a lookup?
-    Bindings::iterator i = extractAttrNameFromPrimopCall(
+    Bindings::iterator i = getAttr(
         state,
         "getAttr",
         attr,
@@ -2191,7 +2191,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         Value & v2(*args[0]->listElems()[i]);
         state.forceAttrs(v2, pos);
 
-        Bindings::iterator j = extractAttrNameFromPrimopCall(
+        Bindings::iterator j = getAttr(
             state,
             "listToAttrs",
             state.sName,
@@ -2203,7 +2203,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
-            Bindings::iterator j2 = extractAttrNameFromPrimopCall(
+            Bindings::iterator j2 = getAttr(
                 state,
                 "listToAttrs",
                 state.sValue,

From 15f4d4fd437070e898982c510228b6e8eea17b7d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 15 Apr 2021 13:51:00 +0200
Subject: [PATCH 863/882] Drop libbz2 / zlib / lzma dependency + style fixes

---
 Makefile.config.in         |  1 -
 configure.ac               | 15 ----------
 flake.nix                  |  2 +-
 src/libutil/compression.cc | 60 +++++++++++++++++++++++---------------
 src/libutil/compression.hh |  6 ++--
 src/libutil/local.mk       |  2 +-
 src/libutil/tarfile.cc     | 37 +++++++++++++----------
 src/libutil/tarfile.hh     | 12 ++++----
 8 files changed, 69 insertions(+), 66 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 3c1f01d1e..fd92365eb 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -15,7 +15,6 @@ LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
diff --git a/configure.ac b/configure.ac
index a24287ff6..c8d59a212 100644
--- a/configure.ac
+++ b/configure.ac
@@ -172,11 +172,6 @@ fi
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
-# Look for libbz2, a required dependency.
-AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
-AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 # Checks for libarchive
 PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
@@ -205,16 +200,6 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
-# Look for liblzma, a required dependency.
-PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
-AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
-  [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
-
-# Look for zlib, a required dependency.
-PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
-AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
-LDFLAGS="-lz $LDFLAGS"
-
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
diff --git a/flake.nix b/flake.nix
index 58dc5019d..b8d5e0087 100644
--- a/flake.nix
+++ b/flake.nix
@@ -82,7 +82,7 @@
 
         buildDeps =
           [ curl
-            bzip2 xz brotli zlib editline
+            bzip2 xz brotli editline
             openssl sqlite
             libarchive
             boost
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 8ba536000..8b50776b3 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -4,8 +4,6 @@
 #include "finally.hh"
 #include "logging.hh"
 
-#include <lzma.h>
-#include <bzlib.h>
 #include <archive.h>
 #include <archive_entry.h>
 #include <cstdio>
@@ -45,12 +43,13 @@ struct ArchiveDecompressionSource : Source
     ArchiveDecompressionSource(Source & src) : src(src) {}
     ~ArchiveDecompressionSource() override {}
     size_t read(char * data, size_t len) override {
-        struct archive_entry* ae;
+        struct archive_entry * ae;
         if (!archive) {
             archive = std::make_unique<TarArchive>(src, true);
-            this->archive->check(archive_read_next_header(this->archive->archive, &ae), "Failed to read header (%s)");
+            this->archive->check(archive_read_next_header(this->archive->archive, &ae),
+                "failed to read header (%s)");
             if (archive_filter_count(this->archive->archive) < 2) {
-                throw CompressionError("Input compression not recognized.");
+                throw CompressionError("input compression not recognized");
             }
         }
         ssize_t result = archive_read_data(this->archive->archive, data, len);
@@ -58,18 +57,20 @@ struct ArchiveDecompressionSource : Source
         if (result == 0) {
             throw EndOfFile("reached end of compressed file");
         }
-        this->archive->check(result, "Failed to read compressed data (%s)");
+        this->archive->check(result, "failed to read compressed data (%s)");
         return result;
     }
 };
+
 struct ArchiveCompressionSink : CompressionSink
 {
     Sink & nextSink;
-    struct archive* archive;
+    struct archive * archive;
+
     ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
         archive = archive_write_new();
         if (!archive) throw Error("failed to initialize libarchive");
-        check(archive_write_add_filter_by_name(archive, format.c_str()), "Couldn't initialize compression (%s)");
+        check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
         check(archive_write_set_format_raw(archive));
         if (format == "xz" && parallel) {
             check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
@@ -80,34 +81,46 @@ struct ArchiveCompressionSink : CompressionSink
         check(archive_write_set_bytes_in_last_block(archive, 1));
         open();
     }
-    ~ArchiveCompressionSink() override {
+
+    ~ArchiveCompressionSink() override
+    {
         if (archive) archive_write_free(archive);
     }
-    void finish() override {
+
+    void finish() override
+    {
         flush();
         check(archive_write_close(archive));
     }
-    void check(int err, const char *reason="Failed to compress (%s)") {
+
+    void check(int err, const std::string & reason = "failed to compress (%s)")
+    {
         if (err == ARCHIVE_EOF)
             throw EndOfFile("reached end of archive");
         else if (err != ARCHIVE_OK)
             throw Error(reason, archive_error_string(this->archive));
     }
-    void write(std::string_view data) override {
+
+    void write(std::string_view data) override
+    {
         ssize_t result = archive_write_data(archive, data.data(), data.length());
         if (result <= 0) check(result);
     }
+
 private:
-    void open() {
-        check(archive_write_open(archive, this, NULL, ArchiveCompressionSink::callback_write, NULL));
-        struct archive_entry *ae = archive_entry_new();
+    void open()
+    {
+        check(archive_write_open(archive, this, nullptr, ArchiveCompressionSink::callback_write, nullptr));
+        auto ae = archive_entry_new();
         archive_entry_set_filetype(ae, AE_IFREG);
         check(archive_write_header(archive, ae));
         archive_entry_free(ae);
     }
-    static ssize_t callback_write(struct archive *archive, void *_self, const void *buffer, size_t length) {
-        ArchiveCompressionSink *self = (ArchiveCompressionSink *)_self;
-        self->nextSink({(const char*)buffer, length});
+
+    static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length)
+    {
+        auto self = (ArchiveCompressionSink *) _self;
+        self->nextSink({(const char *) buffer, length});
         return length;
     }
 };
@@ -193,17 +206,17 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
     else if (method == "br")
         return std::make_unique<BrotliDecompressionSink>(nextSink);
     else
-		return sourceToSink([&](Source & source) {
-			auto decompressionSource = makeDecompressionSource(source);
-			decompressionSource->drainInto(nextSink);
-		});
+        return sourceToSink([&](Source & source) {
+            auto decompressionSource = makeDecompressionSource(source);
+            decompressionSource->drainInto(nextSink);
+        });
 }
 
 struct BrotliCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
     uint8_t outbuf[BUFSIZ];
-    BrotliEncoderState *state;
+    BrotliEncoderState * state;
     bool finished = false;
 
     BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
@@ -251,6 +264,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink
         }
     }
 };
+
 std::unique_ptr<Source> makeDecompressionSource(Source & prev) {
     return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev));
 }
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index 192cb3e91..a140f8be2 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -10,9 +10,9 @@ namespace nix {
 
 struct CompressionSink : BufferedSink, FinishSink
 {
-	using BufferedSink::operator ();
-	using BufferedSink::write;
-	using FinishSink::finish;
+    using BufferedSink::operator ();
+    using BufferedSink::write;
+    using FinishSink::finish;
 };
 
 std::unique_ptr<Source> makeDecompressionSource(Source & prev);
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 5341c58e6..3a6415ee3 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,7 +6,7 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS = -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	libutil_LDFLAGS += -lcpuid
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index b5e1cb4c0..24905130d 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -5,38 +5,41 @@
 #include "tarfile.hh"
 
 namespace nix {
-static int callback_open(struct archive *, void *self) {
+
+static int callback_open(struct archive *, void * self)
+{
     return ARCHIVE_OK;
 }
 
-static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) {
-    TarArchive *self = (TarArchive *)_self;
+static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
+{
+    auto self = (TarArchive *) _self;
     *buffer = self->buffer.data();
 
     try {
         return self->source->read((char *) self->buffer.data(), 4096);
     } catch (EndOfFile &) {
         return 0;
-    } catch (std::exception &err) {
+    } catch (std::exception & err) {
         archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
-
         return -1;
     }
 }
 
-static int callback_close(struct archive *, void *self) {
+static int callback_close(struct archive *, void * self)
+{
     return ARCHIVE_OK;
 }
 
-void TarArchive::check(int err, const char *reason)
+void TarArchive::check(int err, const std::string & reason)
 {
-        if (err == ARCHIVE_EOF)
-            throw EndOfFile("reached end of archive");
-        else if (err != ARCHIVE_OK)
-            throw Error(reason, archive_error_string(this->archive));
-    }
+    if (err == ARCHIVE_EOF)
+        throw EndOfFile("reached end of archive");
+    else if (err != ARCHIVE_OK)
+        throw Error(reason, archive_error_string(this->archive));
+}
 
-TarArchive::TarArchive(Source& source, bool raw) : buffer(4096)
+TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
 {
     this->archive = archive_read_new();
     this->source = &source;
@@ -53,7 +56,7 @@ TarArchive::TarArchive(Source& source, bool raw) : buffer(4096)
 }
 
 
-TarArchive::TarArchive(const Path &path)
+TarArchive::TarArchive(const Path & path)
 {
     this->archive = archive_read_new();
 
@@ -62,11 +65,13 @@ TarArchive::TarArchive(const Path &path)
     check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
 }
 
-void TarArchive::close() {
+void TarArchive::close()
+{
     check(archive_read_close(this->archive), "Failed to close archive (%s)");
 }
 
-TarArchive::~TarArchive() {
+TarArchive::~TarArchive()
+{
     if (this->archive) archive_read_free(this->archive);
 }
 
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index 18adf3490..4d9141fd4 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -4,18 +4,18 @@
 namespace nix {
 
 struct TarArchive {
-    struct archive *archive;
-    Source *source;
+    struct archive * archive;
+    Source * source;
     std::vector<unsigned char> buffer;
 
-    void check(int err, const char *reason = "Failed to extract archive (%s)");
+    void check(int err, const std::string & reason = "failed to extract archive (%s)");
 
-    TarArchive(Source& source, bool raw = false);
+    TarArchive(Source & source, bool raw = false);
 
-    TarArchive(const Path &path);
+    TarArchive(const Path & path);
 
     // disable copy constructor
-    TarArchive(const TarArchive&) = delete;
+    TarArchive(const TarArchive &) = delete;
 
     void close();
 

From 4cd9bd19cd8fecf1dd4cff1f0178091da2ef1316 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 15 Apr 2021 13:57:04 +0200
Subject: [PATCH 864/882] Drop unused tar dependency

---
 configure.ac | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/configure.ac b/configure.ac
index c8d59a212..6c36787f3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -285,17 +285,6 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
-# Do we have GNU tar?
-AC_MSG_CHECKING([if you have a recent GNU tar])
-if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then
-    AC_MSG_RESULT(yes)
-    tarFlags="--warning=no-timestamp"
-else
-    AC_MSG_RESULT(no)
-fi
-AC_SUBST(tarFlags)
-
-
 AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   [path of a statically-linked shell to use as /bin/sh in sandboxes]),
   sandbox_shell=$withval)

From 6905291dafba44ed414f4bc800ac0e09db5cbf9b Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 3 Apr 2021 12:04:57 +0200
Subject: [PATCH 865/882] libcmd/installables: force re-evaluation of cached
 failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I think that it's not very helpful to get "cached failures" in a wrong
`flake.nix`. This can be very confusing when debugging a Nix expression.
See for instance NixOS/nixpkgs#118115.

In fact, the eval cache allows a forced reevaluation which is used for
e.g. `nix eval`.

This change makes sure that this is the case for `nix build` as well. So
rather than

    λ ma27 [~/Projects/exp] → ../nix/outputs/out/bin/nix build -L --rebuild --experimental-features 'nix-command flakes'
    error: cached failure of attribute 'defaultPackage.x86_64-linux'

the evaluation of already-evaluated (and failed) attributes looks like
this now:

    λ ma27 [~/Projects/exp] → ../nix/outputs/out/bin/nix build -L --rebuild --experimental-features 'nix-command flakes'
    error: attribute 'hell' missing

           at /nix/store/mrnvi9ss8zn5wj6gpn4bcd68vbh42mfh-source/flake.nix:6:35:

                5|
                6|     packages.x86_64-linux.hello = nixpkgs.legacyPackages.x86_64-linux.hell;
                 |                                   ^
                7|
    (use '--show-trace' to show detailed location information)
---
 src/libcmd/installables.cc | 6 +++++-
 src/libexpr/eval-cache.cc  | 4 ++--
 src/libexpr/eval-cache.hh  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 5d3026c1a..e15addfae 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -503,7 +503,11 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
-        auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
+        auto attr = root->findAlongAttrPath(
+            parseAttrPath(*state, attrPath),
+            true
+        );
+
         if (!attr) continue;
 
         if (!attr->isDerivation())
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 98d91c905..d7e21783d 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -486,11 +486,11 @@ std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
     return getAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        res = res->maybeGetAttr(attr);
+        res = res->maybeGetAttr(attr, force);
         if (!res) return {};
     }
     return res;
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index e23e45c94..43b34ebcb 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -102,7 +102,7 @@ public:
 
     std::shared_ptr<AttrCursor> getAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
+    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
 
     std::string getString();
 

From 9ac6534f7c39db34d872520b6341bcaa9b2c4a39 Mon Sep 17 00:00:00 2001
From: Alyssa Ross <hi@alyssa.is>
Date: Mon, 19 Apr 2021 18:31:58 +0000
Subject: [PATCH 866/882] Include sys/wait.h everywhere WIFEXITED etc is used

This is required on NetBSD, and I think FreeBSD too.
---
 src/libfetchers/git.cc                | 1 +
 src/libstore/build/derivation-goal.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index b9a240b13..d8e0dbe0a 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -6,6 +6,7 @@
 #include "url-parts.hh"
 
 #include <sys/time.h>
+#include <sys/wait.h>
 
 using namespace std::string_literals;
 
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3ce538f77..55ced2215 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -20,6 +20,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
+#include <sys/wait.h>
 #include <netdb.h>
 #include <fcntl.h>
 #include <termios.h>

From 8d651a1f68c018b8a10dd37da81e9d3612073656 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Apr 2021 22:54:49 +0200
Subject: [PATCH 867/882] Fix brotli compression of files > 128 KiB

This has been broken since faa31f40846f7a4dbc2487d000b112a6aef69d1b.
---
 src/libutil/compression.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 8b50776b3..08812ef57 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -28,7 +28,7 @@ struct ChunkedCompressionSink : CompressionSink
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
         while (!data.empty()) {
             size_t n = std::min(CHUNK_SIZE, data.size());
-            writeInternal(data);
+            writeInternal(data.substr(0, n));
             data.remove_prefix(n);
         }
     }
@@ -265,7 +265,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink
     }
 };
 
-std::unique_ptr<Source> makeDecompressionSource(Source & prev) {
+std::unique_ptr<Source> makeDecompressionSource(Source & prev)
+{
     return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev));
 }
 

From 8d66f5f1107fe87f70ea24ade045720235cc31fa Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 21 Apr 2021 17:33:21 +0200
Subject: [PATCH 868/882] Make `nix shell` fallback to static outputs when
 needed

In case of pure input-addressed derivations, the build loop doesn't
guaranty that the realisations are stored in the db (if the output paths
are already there or can be substituted, the realisations won't be
registered). This caused `nix shell` to fail in some cases because it
was assuming that the realisations were always existing.

A better (but more involved) fix would probably to ensure that we always
register the realisations, but in the meantime this patches the surface
issue.

Fix #4721
---
 src/libcmd/installables.cc | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index e15addfae..39162a662 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -736,9 +736,18 @@ std::set<RealisedPath> toRealisedPaths(
                                     output.first);
                             auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
                             auto realisation = store->queryRealisation(outputId);
-                            if (!realisation)
-                                throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
-                            res.insert(RealisedPath{*realisation});
+                            if (!realisation) {
+                                // TODO: remove this check once #4725 is fixed
+                                // as we’ll have the guaranty that if
+                                // `output.second` exists, then the realisation
+                                // will be there too
+                                if (output.second)
+                                    res.insert(*output.second);
+                                else
+                                    throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
+                            } else {
+                                res.insert(RealisedPath{*realisation});
+                            }
                         }
                         else {
                             // If ca-derivations isn't enabled, behave as if

From 428062e578243336abeacb664bf3dba2674a546f Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 21 Apr 2021 15:26:14 -0500
Subject: [PATCH 869/882] Get deriver when passing --derivation

This makes Nix look up paths derivations when they are passed as a
store paths. So:

$ nix path-info --derivation /nix/store/0pisd259nldh8yfjvw663mspm60cn5v8-hello-2.10

now gives

/nix/store/qp724w90516n4bk5r9gfb37vzmqdh3z7-hello-2.10.drv

instead of "".

If no deriver is available, Nix now errors instead of silently
ignoring that argument.
---
 src/libcmd/installables.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index e15addfae..06ef4c669 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -754,10 +754,8 @@ std::set<RealisedPath> toRealisedPaths(
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
-        for (auto & i : installables)
-            for (auto & b : i->toDerivedPathsWithHints())
-                if (auto bfd = std::get_if<DerivedPathWithHints::Built>(&b))
-                    res.insert(bfd->drvPath);
+        auto drvPaths = toDerivations(store, installables, true);
+        res.insert(drvPaths.begin(), drvPaths.end());
     }
 
     return res;

From ac38c4cdc20a1abc012fae117ad7738093e2919a Mon Sep 17 00:00:00 2001
From: Antoine Martin <antoine@alarsyo.net>
Date: Thu, 22 Apr 2021 03:07:57 +0200
Subject: [PATCH 870/882] Fix typo in flake doc

---
 src/nix/flake.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/flake.md b/src/nix/flake.md
index 0035195e5..9e936a049 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -395,7 +395,7 @@ the `nixpkgs` input of the top-level flake to be equal to the
 `nixpkgs` input of the `dwarffs` input of the top-level flake:
 
 ```nix
-inputs.nixops.follows = "dwarffs/nixpkgs";
+inputs.nixpkgs.follows = "dwarffs/nixpkgs";
 ```
 
 The value of the `follows` attribute is a `/`-separated sequence of

From 97dde3cdd983b2fef9e45dd0ca8d62716a62417d Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <git@lukegb.com>
Date: Thu, 22 Apr 2021 02:33:59 +0000
Subject: [PATCH 871/882] libutil: allow decompression with none/empty method

The S3 store relies on the ability to be able to decompress things with
an empty method, because it just passes the value of the Content-Encoding
directly to decompress.

If the file is not compressed, then this will cause the compression
routine to get confused.

This caused NixOS/nixpkgs#120120.
---
 src/libutil/compression.cc       |  4 +++-
 src/libutil/tests/compression.cc | 18 ++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 08812ef57..c5dae42fa 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -186,7 +186,9 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
 
 ref<std::string> decompress(const std::string & method, const std::string & in)
 {
-    if (method == "br") {
+    if (method == "none" || method == "")
+        return make_ref<std::string>(in);
+    else if (method == "br") {
         StringSink ssink;
         auto sink = makeDecompressionSink(method, ssink);
         (*sink)(in);
diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc
index 5b7a2c5b9..2efa3266b 100644
--- a/src/libutil/tests/compression.cc
+++ b/src/libutil/tests/compression.cc
@@ -17,6 +17,24 @@ namespace nix {
         ASSERT_EQ(*o, "this-is-a-test");
     }
 
+    TEST(decompress, decompressNoneCompressed) {
+        auto method = "none";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, str);
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressEmptyCompressed) {
+        // Empty-method decompression used e.g. by S3 store
+        // (Content-Encoding == "").
+        auto method = "";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, str);
+
+        ASSERT_EQ(*o, str);
+    }
+
     TEST(decompress, decompressXzCompressed) {
         auto method = "xz";
         auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";

From 53857552278173641c0489a3196c51aab769bac8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Apr 2021 10:23:20 +0200
Subject: [PATCH 872/882] Remove makeDecompressionSource()

This function doesn't support all compression methods (i.e. 'none' and
'br') so it shouldn't be exposed.

Also restore the original decompress() as a wrapper around
makeDecompressionSink().
---
 src/libutil/compression.cc | 25 ++++++-------------------
 src/libutil/compression.hh |  2 --
 2 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index c5dae42fa..7e725cae1 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -186,19 +186,11 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
 
 ref<std::string> decompress(const std::string & method, const std::string & in)
 {
-    if (method == "none" || method == "")
-        return make_ref<std::string>(in);
-    else if (method == "br") {
-        StringSink ssink;
-        auto sink = makeDecompressionSink(method, ssink);
-        (*sink)(in);
-        sink->finish();
-        return ssink.s;
-    } else {
-        StringSource ssrc(in);
-        auto src = makeDecompressionSource(ssrc);
-        return make_ref<std::string>(src->drain());
-    }
+    StringSink ssink;
+    auto sink = makeDecompressionSink(method, ssink);
+    (*sink)(in);
+    sink->finish();
+    return ssink.s;
 }
 
 std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
@@ -209,7 +201,7 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
         return std::make_unique<BrotliDecompressionSink>(nextSink);
     else
         return sourceToSink([&](Source & source) {
-            auto decompressionSource = makeDecompressionSource(source);
+            auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source);
             decompressionSource->drainInto(nextSink);
         });
 }
@@ -267,11 +259,6 @@ struct BrotliCompressionSink : ChunkedCompressionSink
     }
 };
 
-std::unique_ptr<Source> makeDecompressionSource(Source & prev)
-{
-    return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev));
-}
-
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
     std::vector<std::string> la_supports = {
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index a140f8be2..338a0d9f2 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -15,8 +15,6 @@ struct CompressionSink : BufferedSink, FinishSink
     using FinishSink::finish;
 };
 
-std::unique_ptr<Source> makeDecompressionSource(Source & prev);
-
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
 std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);

From b1711071d1a22a1c28448823634b0dc7212e4fe5 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 22 Apr 2021 17:54:02 +0200
Subject: [PATCH 873/882] Add a regression test for #4725

---
 tests/ca/substitute.sh | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
index b44fe499a..5d0fc6a87 100644
--- a/tests/ca/substitute.sh
+++ b/tests/ca/substitute.sh
@@ -8,7 +8,8 @@ sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_D
 
 rm -rf $TEST_ROOT/binary_cache
 
-export REMOTE_STORE=file://$TEST_ROOT/binary_cache
+export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache
+export REMOTE_STORE=file://$REMOTE_STORE_DIR
 
 buildDrvs () {
     nix build --file ./content-addressed.nix -L --no-link "$@"
@@ -22,3 +23,20 @@ buildDrvs --post-build-hook ../push-to-store.sh
 clearStore
 buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
 
+# Same thing, but
+# 1. With non-ca derivations
+# 2. Erasing the realisations on the remote store
+#
+# Even in that case, realising the derivations should still produce the right
+# realisations on the local store
+#
+# Regression test for #4725
+clearStore
+nix build --file ../simple.nix -L --no-link --post-build-hook ../push-to-store.sh
+clearStore
+rm -r "$REMOTE_STORE_DIR/realisations"
+nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
+if [[ $(sqlite3 "$NIX_STATE_DIR/db/db.sqlite" 'select count(*) from Realisations') -eq 0 ]]; then
+    echo "Realisations not rebuilt"
+    exit 1
+fi

From 9161e02039e63327830128a3e69034b3cd46c109 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 22 Apr 2021 17:55:36 +0200
Subject: [PATCH 874/882] Always register the realisations of input-addressed
 drvs

Fix #4725
---
 src/libstore/build/derivation-goal.cc | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 55ced2215..9100d3333 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1269,12 +1269,23 @@ void DerivationGoal::checkPathValidity()
             };
         }
         if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-            if (auto real = worker.store.queryRealisation(
-                    DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) {
+            auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
+            if (auto real = worker.store.queryRealisation(drvOutput)) {
                 info.known = {
                     .path = real->outPath,
                     .status = PathStatus::Valid,
                 };
+            } else if (info.known && info.known->status == PathStatus::Valid) {
+                // We know the output because it' a static output of the
+                // derivation, and the output path is valid, but we don't have
+                // its realisation stored (probably because it has been built
+                // without the `ca-derivations` experimental flag)
+                worker.store.registerDrvOutput(
+                    Realisation{
+                        drvOutput,
+                        info.known->path,
+                    }
+                );
             }
         }
     }

From 6ea9c65aece0b65825c99738340ea4944d083234 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 23 Apr 2021 09:34:16 +0200
Subject: [PATCH 875/882] fixup! Add a regression test for #4725

---
 tests/ca/substitute.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
index 5d0fc6a87..737c851a5 100644
--- a/tests/ca/substitute.sh
+++ b/tests/ca/substitute.sh
@@ -36,7 +36,12 @@ nix build --file ../simple.nix -L --no-link --post-build-hook ../push-to-store.s
 clearStore
 rm -r "$REMOTE_STORE_DIR/realisations"
 nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
-if [[ $(sqlite3 "$NIX_STATE_DIR/db/db.sqlite" 'select count(*) from Realisations') -eq 0 ]]; then
+# There's no easy way to check whether a realisation is present on the local
+# store − short of manually querying the db, but the build environment doesn't
+# have the sqlite binary − so we instead push things again, and check that the
+# realisations have correctly been pushed to the remote store
+nix copy --to "$REMOTE_STORE" --file ../simple.nix
+if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
     echo "Realisations not rebuilt"
     exit 1
 fi

From d3df747cb6813fc5f9d0fcc63d6fd16292c92e04 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 23 Apr 2021 09:34:43 +0200
Subject: [PATCH 876/882] Revert "Make `nix shell` fallback to static outputs
 when needed"

This reverts commit 8d66f5f1107fe87f70ea24ade045720235cc31fa.

This fix isn't needed anymore now that the realisations are always
properly registered
---
 src/libcmd/installables.cc | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 10855688c..06ef4c669 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -736,18 +736,9 @@ std::set<RealisedPath> toRealisedPaths(
                                     output.first);
                             auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
                             auto realisation = store->queryRealisation(outputId);
-                            if (!realisation) {
-                                // TODO: remove this check once #4725 is fixed
-                                // as we’ll have the guaranty that if
-                                // `output.second` exists, then the realisation
-                                // will be there too
-                                if (output.second)
-                                    res.insert(*output.second);
-                                else
-                                    throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
-                            } else {
-                                res.insert(RealisedPath{*realisation});
-                            }
+                            if (!realisation)
+                                throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
+                            res.insert(RealisedPath{*realisation});
                         }
                         else {
                             // If ca-derivations isn't enabled, behave as if

From 2207018df574e4b15e552a9ce174f13cc024b6a5 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 23 Apr 2021 10:28:26 +0200
Subject: [PATCH 877/882] (briefly) document how to build ca derivations

---
 doc/manual/src/expressions/advanced-attributes.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
index 31ebadda1..1b3c69da2 100644
--- a/doc/manual/src/expressions/advanced-attributes.md
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -207,6 +207,14 @@ Derivations can declare some infrequently used optional attributes.
     the hash in either hexadecimal or base-32 notation. (See the
     [`nix-hash` command](../command-ref/nix-hash.md) for information
     about converting to and from base-32 notation.)
+    
+  - `__contentAddressed`
+    If this **experimental** attribute is set to true, then the derivation
+    outputs will be stored in a content-addressed location rather than the
+    traditional input-addressed one.
+    This only has an effect if the `ca-derivation` experimental feature is enabled.
+    
+    Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above).
 
   - `passAsFile`  
     A list of names of attributes that should be passed via files rather

From 29132f99c56555c323c28ae046d58c0cddf2e221 Mon Sep 17 00:00:00 2001
From: Alyssa Ross <hi@alyssa.is>
Date: Fri, 23 Apr 2021 10:52:54 +0000
Subject: [PATCH 878/882] doc: fix section in nix3 man page metadata

These man pages said they were in section 7, even though we were
installing them to section 1 (which is the right place for them).
---
 doc/manual/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 81a7755e8..b9cbe48f6 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -80,7 +80,7 @@ install: $(d)/src/command-ref/new-cli
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
 	  printf "Title: %s\n\n" "$$name" > $$i.tmp; \
 	  cat $$i >> $$i.tmp; \
-	  lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \
+	  lowdown -sT man -M section=1 $$i.tmp -o $(mandir)/man1/$$name.1; \
 	done
 
 $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md

From e65f604ea9e4753cb995f93018d1426df0498565 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 23 Apr 2021 14:14:15 +0200
Subject: [PATCH 879/882] Set more man sections

---
 doc/manual/local.mk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index b9cbe48f6..271529b38 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -25,19 +25,19 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -
 $(d)/%.1: $(d)/src/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
 	@cat $^ >> $^.tmp
-	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	$(trace-gen) lowdown -sT man -M section=1 $^.tmp -o $@
 	@rm $^.tmp
 
 $(d)/%.8: $(d)/src/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp
 	@cat $^ >> $^.tmp
-	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	$(trace-gen) lowdown -sT man -M section=8 $^.tmp -o $@
 	@rm $^.tmp
 
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
-	$(trace-gen) lowdown -sT man $^.tmp -o $@
+	$(trace-gen) lowdown -sT man -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
 $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli

From 788008385ef5bf7edb799977525d6f73f02c76bc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 23 Apr 2021 14:14:34 +0200
Subject: [PATCH 880/882] Use lowdown 0.8.4

---
 flake.lock | 18 ++++++++++++++++++
 flake.nix  | 10 ++++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index 9867e694b..06c507e7d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,22 @@
 {
   "nodes": {
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1617481909,
+        "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "ref": "VERSION_0_8_4",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1614309161,
@@ -17,6 +34,7 @@
     },
     "root": {
       "inputs": {
+        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs"
       }
     }
diff --git a/flake.nix b/flake.nix
index 8cd000e83..ebaafb049 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,9 +2,9 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small";
-  #inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
+  inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; };
 
-  outputs = { self, nixpkgs }:
+  outputs = { self, nixpkgs, lowdown-src }:
 
     let
 
@@ -271,14 +271,16 @@
         };
 
         lowdown = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.8.0";
+          name = "lowdown-0.8.4";
 
+          /*
           src = fetchurl {
             url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
             hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
           };
+          */
 
-          #src = lowdown-src;
+          src = lowdown-src;
 
           outputs = [ "out" "bin" "dev" ];
 

From 31313d1401ffe387e69b84c62ba8798bf23a8914 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 23 Apr 2021 14:30:42 +0200
Subject: [PATCH 881/882] Replace the trailing markdown spaces by a backslash
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

They are equivalent according to
<https://spec.commonmark.org/0.29/#hard-line-breaks>,
and the trailing spaces tend to be a pain (because the make git
complain, editors tend to want to remove them − the `.editorconfig`
actually specifies that − etc..).
---
 .../src/advanced-topics/cores-vs-jobs.md      |  4 +-
 doc/manual/src/command-ref/env-common.md      | 30 +++---
 doc/manual/src/command-ref/nix-build.md       |  6 +-
 doc/manual/src/command-ref/nix-channel.md     | 16 ++--
 .../src/command-ref/nix-copy-closure.md       | 14 +--
 doc/manual/src/command-ref/nix-env.md         | 76 +++++++--------
 doc/manual/src/command-ref/nix-hash.md        | 12 +--
 doc/manual/src/command-ref/nix-instantiate.md | 16 ++--
 .../src/command-ref/nix-prefetch-url.md       | 10 +-
 doc/manual/src/command-ref/nix-shell.md       | 16 ++--
 doc/manual/src/command-ref/nix-store.md       | 62 ++++++------
 doc/manual/src/command-ref/opt-common.md      | 96 +++++++++----------
 .../src/expressions/advanced-attributes.md    | 92 +++++++++---------
 .../src/expressions/builtin-constants.md      |  4 +-
 doc/manual/src/expressions/builtins-prefix.md |  2 +-
 doc/manual/src/glossary.md                    | 32 +++----
 .../src/package-management/s3-substituter.md  |  8 +-
 src/libexpr/primops.cc                        | 10 +-
 src/libexpr/primops/fetchTree.cc              | 12 +--
 src/libstore/globals.hh                       |  2 +-
 20 files changed, 260 insertions(+), 260 deletions(-)

diff --git a/doc/manual/src/advanced-topics/cores-vs-jobs.md b/doc/manual/src/advanced-topics/cores-vs-jobs.md
index 4a9058ca1..9e91ab9c7 100644
--- a/doc/manual/src/advanced-topics/cores-vs-jobs.md
+++ b/doc/manual/src/advanced-topics/cores-vs-jobs.md
@@ -4,13 +4,13 @@ Nix has two relevant settings with regards to how your CPU cores will
 be utilized: `cores` and `max-jobs`. This chapter will talk about what
 they are, how they interact, and their configuration trade-offs.
 
-  - `max-jobs`  
+  - `max-jobs`\
     Dictates how many separate derivations will be built at the same
     time. If you set this to zero, the local machine will do no
     builds.  Nix will still substitute from binary caches, and build
     remotely if remote builders are configured.
 
-  - `cores`  
+  - `cores`\
     Suggests how many cores each derivation should use. Similar to
     `make -j`.
 
diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index c670d82b8..b709ca9d1 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -2,11 +2,11 @@
 
 Most Nix commands interpret the following environment variables:
 
-  - `IN_NIX_SHELL`  
+  - `IN_NIX_SHELL`\
     Indicator that tells if the current environment was set up by
     `nix-shell`. Since Nix 2.0 the values are `"pure"` and `"impure"`
 
-  - `NIX_PATH`  
+  - `NIX_PATH`\
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
@@ -40,7 +40,7 @@ Most Nix commands interpret the following environment variables:
     The search path can be extended using the `-I` option, which takes
     precedence over `NIX_PATH`.
 
-  - `NIX_IGNORE_SYMLINK_STORE`  
+  - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
     allowed to contain any symlink components. This is to prevent
     “impure” builds. Builders sometimes “canonicalise” paths by
@@ -62,41 +62,41 @@ Most Nix commands interpret the following environment variables:
     
     Consult the mount 8 manual page for details.
 
-  - `NIX_STORE_DIR`  
+  - `NIX_STORE_DIR`\
     Overrides the location of the Nix store (default `prefix/store`).
 
-  - `NIX_DATA_DIR`  
+  - `NIX_DATA_DIR`\
     Overrides the location of the Nix static data directory (default
     `prefix/share`).
 
-  - `NIX_LOG_DIR`  
+  - `NIX_LOG_DIR`\
     Overrides the location of the Nix log directory (default
     `prefix/var/log/nix`).
 
-  - `NIX_STATE_DIR`  
+  - `NIX_STATE_DIR`\
     Overrides the location of the Nix state directory (default
     `prefix/var/nix`).
 
-  - `NIX_CONF_DIR`  
+  - `NIX_CONF_DIR`\
     Overrides the location of the system Nix configuration directory
     (default `prefix/etc/nix`).
 
-  - `NIX_CONFIG`  
+  - `NIX_CONFIG`\
     Applies settings from Nix configuration from the environment.
     The content is treated as if it was read from a Nix configuration file.
     Settings are separated by the newline character.
 
-  - `NIX_USER_CONF_FILES`  
+  - `NIX_USER_CONF_FILES`\
     Overrides the location of the user Nix configuration files to load
     from (defaults to the XDG spec locations). The variable is treated
     as a list separated by the `:` token.
 
-  - `TMPDIR`  
+  - `TMPDIR`\
     Use the specified directory to store temporary files. In particular,
     this includes temporary build directories; these can take up
     substantial amounts of disk space. The default is `/tmp`.
 
-  - `NIX_REMOTE`  
+  - `NIX_REMOTE`\
     This variable should be set to `daemon` if you want to use the Nix
     daemon to execute Nix operations. This is necessary in [multi-user
     Nix installations](../installation/multi-user.md). If the Nix
@@ -104,16 +104,16 @@ Most Nix commands interpret the following environment variables:
     should be set to `unix://path/to/socket`. Otherwise, it should be
     left unset.
 
-  - `NIX_SHOW_STATS`  
+  - `NIX_SHOW_STATS`\
     If set to `1`, Nix will print some evaluation statistics, such as
     the number of values allocated.
 
-  - `NIX_COUNT_CALLS`  
+  - `NIX_COUNT_CALLS`\
     If set to `1`, Nix will print how often functions were called during
     Nix expression evaluation. This is useful for profiling your Nix
     expressions.
 
-  - `GC_INITIAL_HEAP_SIZE`  
+  - `GC_INITIAL_HEAP_SIZE`\
     If Nix has been configured to use the Boehm garbage collector, this
     variable sets the initial size of the heap in bytes. It defaults to
     384 MiB. Setting it to a low value reduces memory consumption, but
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 4565bfbc2..43de7a6e6 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -47,16 +47,16 @@ All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
 `nix-instantiate`.
 
-  - `--no-out-link`  
+  - `--no-out-link`\
     Do not create a symlink to the output path. Note that as a result
     the output does not become a root of the garbage collector, and so
     might be deleted by `nix-store
                     --gc`.
 
-  - `--dry-run`  
+  - `--dry-run`\
     Show what store paths would be built or downloaded.
 
-  - `--out-link` / `-o` *outlink*  
+  - `--out-link` / `-o` *outlink*\
     Change the name of the symlink to the output path created from
     `result` to *outlink*.
 
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index 4ca12d2cc..24353525f 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -17,26 +17,26 @@ To see the list of official NixOS channels, visit
 
 This command has the following operations:
 
-  - `--add` *url* \[*name*\]  
+  - `--add` *url* \[*name*\]\
     Adds a channel named *name* with URL *url* to the list of subscribed
     channels. If *name* is omitted, it defaults to the last component of
     *url*, with the suffixes `-stable` or `-unstable` removed.
 
-  - `--remove` *name*  
+  - `--remove` *name*\
     Removes the channel named *name* from the list of subscribed
     channels.
 
-  - `--list`  
+  - `--list`\
     Prints the names and URLs of all subscribed channels on standard
     output.
 
-  - `--update` \[*names*…\]  
+  - `--update` \[*names*…\]\
     Downloads the Nix expressions of all subscribed channels (or only
     those included in *names* if specified) and makes them the default
     for `nix-env` operations (by symlinking them from the directory
     `~/.nix-defexpr`).
 
-  - `--rollback` \[*generation*\]  
+  - `--rollback` \[*generation*\]\
     Reverts the previous call to `nix-channel
                     --update`. Optionally, you can specify a specific channel generation
     number to restore.
@@ -70,14 +70,14 @@ $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
 
 # Files
 
-  - `/nix/var/nix/profiles/per-user/username/channels`  
+  - `/nix/var/nix/profiles/per-user/username/channels`\
     `nix-channel` uses a `nix-env` profile to keep track of previous
     versions of the subscribed channels. Every time you run `nix-channel
     --update`, a new channel generation (that is, a symlink to the
     channel Nix expressions in the Nix store) is created. This enables
     `nix-channel --rollback` to revert to previous versions.
 
-  - `~/.nix-defexpr/channels`  
+  - `~/.nix-defexpr/channels`\
     This is a symlink to
     `/nix/var/nix/profiles/per-user/username/channels`. It ensures that
     `nix-env` can find your channels. In a multi-user installation, you
@@ -89,7 +89,7 @@ $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
 A channel URL should point to a directory containing the following
 files:
 
-  - `nixexprs.tar.xz`  
+  - `nixexprs.tar.xz`\
     A tarball containing Nix expressions and files referenced by them
     (such as build scripts and patches). At the top level, the tarball
     should contain a single directory. That directory must contain a
diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index dcb844a72..7047d3012 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -35,21 +35,21 @@ and second to send the dump of those paths.  If this bothers you, use
 
 # Options
 
-  - `--to`  
+  - `--to`\
     Copy the closure of _paths_ from the local Nix store to the Nix
     store on _machine_. This is the default.
 
-  - `--from`  
+  - `--from`\
     Copy the closure of _paths_ from the Nix store on _machine_ to the
     local Nix store.
 
-  - `--gzip`  
+  - `--gzip`\
     Enable compression of the SSH connection.
 
-  - `--include-outputs`  
+  - `--include-outputs`\
     Also copy the outputs of store derivations included in the closure.
 
-  - `--use-substitutes` / `-s`  
+  - `--use-substitutes` / `-s`\
     Attempt to download missing paths on the target machine using Nix’s
     substitute mechanism.  Any paths that cannot be substituted on the
     target are still copied normally from the source.  This is useful,
@@ -58,12 +58,12 @@ and second to send the dump of those paths.  If this bothers you, use
     `nixos.org` (the default binary cache server) is
     fast.
 
-  - `-v`  
+  - `-v`\
     Show verbose output.
 
 # Environment variables
 
-  - `NIX_SSHOPTS`  
+  - `NIX_SSHOPTS`\
     Additional options to be passed to `ssh` on the command
     line.
 
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index 1c23bb0ad..9138fa05a 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -36,27 +36,27 @@ case-sensitive. The regular expression can optionally be followed by a
 dash and a version number; if omitted, any version of the package will
 match. Here are some examples:
 
-  - `firefox`  
+  - `firefox`\
     Matches the package name `firefox` and any version.
 
-  - `firefox-32.0`  
+  - `firefox-32.0`\
     Matches the package name `firefox` and version `32.0`.
 
-  - `gtk\\+`  
+  - `gtk\\+`\
     Matches the package name `gtk+`. The `+` character must be escaped
     using a backslash to prevent it from being interpreted as a
     quantifier, and the backslash must be escaped in turn with another
     backslash to ensure that the shell passes it on.
 
-  - `.\*`  
+  - `.\*`\
     Matches any package name. This is the default for most commands.
 
-  - `'.*zip.*'`  
+  - `'.*zip.*'`\
     Matches any package name containing the string `zip`. Note the dots:
     `'*zip*'` does not work, because in a regular expression, the
     character `*` is interpreted as a quantifier.
 
-  - `'.*(firefox|chromium).*'`  
+  - `'.*(firefox|chromium).*'`\
     Matches any package name containing the strings `firefox` or
     `chromium`.
 
@@ -66,7 +66,7 @@ This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
 have an effect.
 
-  - `--file` / `-f` *path*  
+  - `--file` / `-f` *path*\
     Specifies the Nix expression (designated below as the *active Nix
     expression*) used by the `--install`, `--upgrade`, and `--query
     --available` operations to obtain derivations. The default is
@@ -77,13 +77,13 @@ have an effect.
     unpacked to a temporary location. The tarball must include a single
     top-level directory containing at least a file named `default.nix`.
 
-  - `--profile` / `-p` *path*  
+  - `--profile` / `-p` *path*\
     Specifies the profile to be used by those operations that operate on
     a profile (designated below as the *active profile*). A profile is a
     sequence of user environments called *generations*, one of which is
     the *current generation*.
 
-  - `--dry-run`  
+  - `--dry-run`\
     For the `--install`, `--upgrade`, `--uninstall`,
     `--switch-generation`, `--delete-generations` and `--rollback`
     operations, this flag will cause `nix-env` to print what *would* be
@@ -93,7 +93,7 @@ have an effect.
     [substituted](../glossary.md) (i.e., downloaded) and which paths
     will be built from source (because no substitute is available).
 
-  - `--system-filter` *system*  
+  - `--system-filter` *system*\
     By default, operations such as `--query
                     --available` show derivations matching any platform. This option
     allows you to use derivations for the specified platform *system*.
@@ -102,7 +102,7 @@ have an effect.
 
 # Files
 
-  - `~/.nix-defexpr`  
+  - `~/.nix-defexpr`\
     The source for the default Nix expressions used by the
     `--install`, `--upgrade`, and `--query --available` operations to
     obtain derivations. The `--file` option may be used to override
@@ -140,7 +140,7 @@ have an effect.
     The command `nix-channel` places symlinks to the downloaded Nix
     expressions from each subscribed channel in this directory.
 
-  - `~/.nix-profile`  
+  - `~/.nix-profile`\
     A symbolic link to the user's current profile. By default, this
     symlink points to `prefix/var/nix/profiles/default`. The `PATH`
     environment variable should include `~/.nix-profile/bin` for the
@@ -217,13 +217,13 @@ a number of possible ways:
 
 ## Flags
 
-  - `--prebuilt-only` / `-b`  
+  - `--prebuilt-only` / `-b`\
     Use only derivations for which a substitute is registered, i.e.,
     there is a pre-built binary available that can be downloaded in lieu
     of building the derivation. Thus, no packages will be built from
     source.
 
-  - `--preserve-installed`; `-P`  
+  - `--preserve-installed`; `-P`\
     Do not remove derivations with a name matching one of the
     derivations being installed. Usually, trying to have two versions of
     the same package installed in the same generation of a profile will
@@ -231,7 +231,7 @@ a number of possible ways:
     clashes between the two versions. However, this is not the case for
     all packages.
 
-  - `--remove-all`; `-r`  
+  - `--remove-all`; `-r`\
     Remove all previously installed packages first. This is equivalent
     to running `nix-env -e '.*'` first, except that everything happens
     in a single transaction.
@@ -346,24 +346,24 @@ version is installed.
 
 ## Flags
 
-  - `--lt`  
+  - `--lt`\
     Only upgrade a derivation to newer versions. This is the default.
 
-  - `--leq`  
+  - `--leq`\
     In addition to upgrading to newer versions, also “upgrade” to
     derivations that have the same version. Version are not a unique
     identification of a derivation, so there may be many derivations
     that have the same version. This flag may be useful to force
     “synchronisation” between the installed and available derivations.
 
-  - `--eq`  
+  - `--eq`\
     *Only* “upgrade” to derivations that have the same version. This may
     not seem very useful, but it actually is, e.g., when there is a new
     release of Nixpkgs and you want to replace installed applications
     with the same versions built against newer dependencies (to reduce
     the number of dependencies floating around on your system).
 
-  - `--always`  
+  - `--always`\
     In addition to upgrading to newer versions, also “upgrade” to
     derivations that have the same or a lower version. I.e., derivations
     may actually be downgraded depending on what is available in the
@@ -578,11 +578,11 @@ The derivations are sorted by their `name` attributes.
 The following flags specify the set of things on which the query
 operates.
 
-  - `--installed`  
+  - `--installed`\
     The query operates on the store paths that are installed in the
     current generation of the active profile. This is the default.
 
-  - `--available`; `-a`  
+  - `--available`; `-a`\
     The query operates on the derivations that are available in the
     active Nix expression.
 
@@ -593,24 +593,24 @@ selected derivations. Multiple flags may be specified, in which case the
 information is shown in the order given here. Note that the name of the
 derivation is shown unless `--no-name` is specified.
 
-  - `--xml`  
+  - `--xml`\
     Print the result in an XML representation suitable for automatic
     processing by other tools. The root element is called `items`, which
     contains a `item` element for each available or installed
     derivation. The fields discussed below are all stored in attributes
     of the `item` elements.
 
-  - `--json`  
+  - `--json`\
     Print the result in a JSON representation suitable for automatic
     processing by other tools.
 
-  - `--prebuilt-only` / `-b`  
+  - `--prebuilt-only` / `-b`\
     Show only derivations for which a substitute is registered, i.e.,
     there is a pre-built binary available that can be downloaded in lieu
     of building the derivation. Thus, this shows all packages that
     probably can be installed quickly.
 
-  - `--status`; `-s`  
+  - `--status`; `-s`\
     Print the *status* of the derivation. The status consists of three
     characters. The first is `I` or `-`, indicating whether the
     derivation is currently installed in the current generation of the
@@ -621,49 +621,49 @@ derivation is shown unless `--no-name` is specified.
     derivation to be built. The third is `S` or `-`, indicating whether
     a substitute is available for the derivation.
 
-  - `--attr-path`; `-P`  
+  - `--attr-path`; `-P`\
     Print the *attribute path* of the derivation, which can be used to
     unambiguously select it using the `--attr` option available in
     commands that install derivations like `nix-env --install`. This
     option only works together with `--available`
 
-  - `--no-name`  
+  - `--no-name`\
     Suppress printing of the `name` attribute of each derivation.
 
-  - `--compare-versions` / `-c`  
+  - `--compare-versions` / `-c`\
     Compare installed versions to available versions, or vice versa (if
     `--available` is given). This is useful for quickly seeing whether
     upgrades for installed packages are available in a Nix expression. A
     column is added with the following meaning:
 
-      - `<` *version*  
+      - `<` *version*\
         A newer version of the package is available or installed.
 
-      - `=` *version*  
+      - `=` *version*\
         At most the same version of the package is available or
         installed.
 
-      - `>` *version*  
+      - `>` *version*\
         Only older versions of the package are available or installed.
 
-      - `- ?`  
+      - `- ?`\
         No version of the package is available or installed.
 
-  - `--system`  
+  - `--system`\
     Print the `system` attribute of the derivation.
 
-  - `--drv-path`  
+  - `--drv-path`\
     Print the path of the store derivation.
 
-  - `--out-path`  
+  - `--out-path`\
     Print the output path of the derivation.
 
-  - `--description`  
+  - `--description`\
     Print a short (one-line) description of the derivation, if
     available. The description is taken from the `meta.description`
     attribute of the derivation.
 
-  - `--meta`  
+  - `--meta`\
     Print all of the meta-attributes of the derivation. This option is
     only available with `--xml` or `--json`.
 
@@ -874,7 +874,7 @@ error: no generation older than the current (91) exists
 
 # Environment variables
 
-  - `NIX_PROFILE`  
+  - `NIX_PROFILE`\
     Location of the Nix profile. Defaults to the target of the symlink
     `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default`
     otherwise.
diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md
index de0459b9e..45f67f1c5 100644
--- a/doc/manual/src/command-ref/nix-hash.md
+++ b/doc/manual/src/command-ref/nix-hash.md
@@ -29,29 +29,29 @@ md5sum`.
 
 # Options
 
-  - `--flat`  
+  - `--flat`\
     Print the cryptographic hash of the contents of each regular file
     *path*. That is, do not compute the hash over the dump of *path*.
     The result is identical to that produced by the GNU commands
     `md5sum` and `sha1sum`.
 
-  - `--base32`  
+  - `--base32`\
     Print the hash in a base-32 representation rather than hexadecimal.
     This base-32 representation is more compact and can be used in Nix
     expressions (such as in calls to `fetchurl`).
 
-  - `--truncate`  
+  - `--truncate`\
     Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits.
 
-  - `--type` *hashAlgo*  
+  - `--type` *hashAlgo*\
     Use the specified cryptographic hash algorithm, which can be one of
     `md5`, `sha1`, `sha256`, and `sha512`.
 
-  - `--to-base16`  
+  - `--to-base16`\
     Don’t hash anything, but convert the base-32 hash representation
     *hash* to hexadecimal.
 
-  - `--to-base32`  
+  - `--to-base32`\
     Don’t hash anything, but convert the hexadecimal hash representation
     *hash* to base-32.
 
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index c369397b6..2e198daed 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -29,26 +29,26 @@ standard input.
 
 # Options
 
-  - `--add-root` *path*  
+  - `--add-root` *path*\
     See the [corresponding option](nix-store.md) in `nix-store`.
 
-  - `--parse`  
+  - `--parse`\
     Just parse the input files, and print their abstract syntax trees on
     standard output in ATerm format.
 
-  - `--eval`  
+  - `--eval`\
     Just parse and evaluate the input files, and print the resulting
     values on standard output. No instantiation of store derivations
     takes place.
 
-  - `--find-file`  
+  - `--find-file`\
     Look up the given files in Nix’s search path (as specified by the
     `NIX_PATH` environment variable). If found, print the corresponding
     absolute paths on standard output. For instance, if `NIX_PATH` is
     `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file
     nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`.
 
-  - `--strict`  
+  - `--strict`\
     When used with `--eval`, recursively evaluate list elements and
     attributes. Normally, such sub-expressions are left unevaluated
     (since the Nix expression language is lazy).
@@ -58,17 +58,17 @@ standard input.
     > This option can cause non-termination, because lazy data
     > structures can be infinitely large.
 
-  - `--json`  
+  - `--json`\
     When used with `--eval`, print the resulting value as an JSON
     representation of the abstract syntax tree rather than as an ATerm.
 
-  - `--xml`  
+  - `--xml`\
     When used with `--eval`, print the resulting value as an XML
     representation of the abstract syntax tree rather than as an ATerm.
     The schema is the same as that used by the [`toXML`
     built-in](../expressions/builtins.md).
 
-  - `--read-write-mode`  
+  - `--read-write-mode`\
     When used with `--eval`, perform evaluation in read/write mode so
     nix language features that require it will still work (at the cost
     of needing to do instantiation of every evaluated derivation). If
diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md
index 59ab89b29..3bcd209e2 100644
--- a/doc/manual/src/command-ref/nix-prefetch-url.md
+++ b/doc/manual/src/command-ref/nix-prefetch-url.md
@@ -37,22 +37,22 @@ Nix store is also printed.
 
 # Options
 
-  - `--type` *hashAlgo*  
+  - `--type` *hashAlgo*\
     Use the specified cryptographic hash algorithm, which can be one of
     `md5`, `sha1`, `sha256`, and `sha512`.
 
-  - `--print-path`  
+  - `--print-path`\
     Print the store path of the downloaded file on standard output.
 
-  - `--unpack`  
+  - `--unpack`\
     Unpack the archive (which must be a tarball or zip file) and add the
     result to the Nix store. The resulting hash can be used with
     functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`.
 
-  - `--executable`  
+  - `--executable`\
     Set the executable bit on the downloaded file.
 
-  - `--name` *name*  
+  - `--name` *name*\
     Override the name of the file in the Nix store. By default, this is
     `hash-basename`, where *basename* is the last component of *url*.
     Overriding the name is necessary when *basename* contains characters
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 54812a49f..dcd7cc70c 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -54,7 +54,7 @@ All options not listed here are passed to `nix-store
 --realise`, except for `--arg` and `--attr` / `-A` which are passed to
 `nix-instantiate`.
 
-  - `--command` *cmd*  
+  - `--command` *cmd*\
     In the environment of the derivation, run the shell command *cmd*.
     This command is executed in an interactive shell. (Use `--run` to
     use a non-interactive shell instead.) However, a call to `exit` is
@@ -64,16 +64,16 @@ All options not listed here are passed to `nix-store
     drop you into the interactive shell. This can be useful for doing
     any additional initialisation.
 
-  - `--run` *cmd*  
+  - `--run` *cmd*\
     Like `--command`, but executes the command in a non-interactive
     shell. This means (among other things) that if you hit Ctrl-C while
     the command is running, the shell exits.
 
-  - `--exclude` *regexp*  
+  - `--exclude` *regexp*\
     Do not build any dependencies whose store path matches the regular
     expression *regexp*. This option may be specified multiple times.
 
-  - `--pure`  
+  - `--pure`\
     If this flag is specified, the environment is almost entirely
     cleared before the interactive shell is started, so you get an
     environment that more closely corresponds to the “real” Nix build. A
@@ -82,18 +82,18 @@ All options not listed here are passed to `nix-store
     installation) `/etc/bashrc` is still sourced, so any variables set
     there will affect the interactive shell.
 
-  - `--packages` / `-p` *packages*…  
+  - `--packages` / `-p` *packages*…\
     Set up an environment in which the specified packages are present.
     The command line arguments are interpreted as attribute names inside
     the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk`
     will start a shell in which the packages denoted by the attribute
     names `libjpeg` and `openjdk` are present.
 
-  - `-i` *interpreter*  
+  - `-i` *interpreter*\
     The chained script interpreter to be invoked by `nix-shell`. Only
     applicable in `#!`-scripts (described below).
 
-  - `--keep` *name*  
+  - `--keep` *name*\
     When a `--pure` shell is started, keep the listed environment
     variables.
 
@@ -101,7 +101,7 @@ The following common options are supported:
 
 # Environment variables
 
-  - `NIX_BUILD_SHELL`  
+  - `NIX_BUILD_SHELL`\
     Shell used to start the interactive environment. Defaults to the
     `bash` found in `PATH`.
 
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index 49d06f31e..7a131dc02 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -22,7 +22,7 @@ This section lists the options that are common to all operations. These
 options are allowed for every subcommand, though they may not always
 have an effect.
 
-  - `--add-root` *path*  
+  - `--add-root` *path*\
     Causes the result of a realisation (`--realise` and
     `--force-realise`) to be registered as a root of the garbage
     collector. *path* will be created as a symlink to the resulting
@@ -86,15 +86,15 @@ non-derivations argument, the argument itself is printed.)
 
 The following flags are available:
 
-  - `--dry-run`  
+  - `--dry-run`\
     Print on standard error a description of what packages would be
     built or downloaded, without actually performing the operation.
 
-  - `--ignore-unknown`  
+  - `--ignore-unknown`\
     If a non-derivation path does not have a substitute, then silently
     ignore it.
 
-  - `--check`  
+  - `--check`\
     This option allows you to check whether a derivation is
     deterministic. It rebuilds the specified derivation and checks
     whether the result is bitwise-identical with the existing outputs,
@@ -110,20 +110,20 @@ The following flags are available:
 
 Special exit codes:
 
-  - `100`  
+  - `100`\
     Generic build failure, the builder process returned with a non-zero
     exit code.
 
-  - `101`  
+  - `101`\
     Build timeout, the build was aborted because it did not complete
     within the specified `timeout`.
 
-  - `102`  
+  - `102`\
     Hash mismatch, the build output was rejected because it does not
     match the [`outputHash` attribute of the
     derivation](../expressions/advanced-attributes.md).
 
-  - `104`  
+  - `104`\
     Not deterministic, the build succeeded in check mode but the
     resulting output is not binary reproducable.
 
@@ -170,7 +170,7 @@ access to a restricted ssh user.
 
 The following flags are available:
 
-  - `--write`  
+  - `--write`\
     Allow the connected client to request the realization of
     derivations. In effect, this can be used to make the host act as a
     remote builder.
@@ -200,18 +200,18 @@ reachable via file system references from a set of “roots”, are deleted.
 
 The following suboperations may be specified:
 
-  - `--print-roots`  
+  - `--print-roots`\
     This operation prints on standard output the set of roots used by
     the garbage collector.
 
-  - `--print-live`  
+  - `--print-live`\
     This operation prints on standard output the set of “live” store
     paths, which are all the store paths reachable from the roots. Live
     paths should never be deleted, since that would break consistency —
     it would become possible that applications are installed that
     reference things that are no longer present in the store.
 
-  - `--print-dead`  
+  - `--print-dead`\
     This operation prints out on standard output the set of “dead” store
     paths, which is just the opposite of the set of live paths: any path
     in the store that is not live (with respect to the roots) is dead.
@@ -219,7 +219,7 @@ The following suboperations may be specified:
 By default, all unreachable paths are deleted. The following options
 control what gets deleted and in what order:
 
-  - `--max-freed` *bytes*  
+  - `--max-freed` *bytes*\
     Keep deleting paths until at least *bytes* bytes have been deleted,
     then stop. The argument *bytes* can be followed by the
     multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB
@@ -300,22 +300,22 @@ symlink.
 
 ## Common query options
 
-  - `--use-output`; `-u`  
+  - `--use-output`; `-u`\
     For each argument to the query that is a store derivation, apply the
     query to the output path of the derivation instead.
 
-  - `--force-realise`; `-f`  
+  - `--force-realise`; `-f`\
     Realise each argument to the query first (see [`nix-store
     --realise`](#operation---realise)).
 
 ## Queries
 
-  - `--outputs`  
+  - `--outputs`\
     Prints out the [output paths](../glossary.md) of the store
     derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
-  - `--requisites`; `-R`  
+  - `--requisites`; `-R`\
     Prints out the [closure](../glossary.md) of the store path *paths*.
 
     This query has one option:
@@ -332,31 +332,31 @@ symlink.
     dependencies) is obtained by distributing the closure of a store
     derivation and specifying the option `--include-outputs`.
 
-  - `--references`  
+  - `--references`\
     Prints the set of [references](../glossary.md) of the store paths
     *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
-  - `--referrers`  
+  - `--referrers`\
     Prints the set of *referrers* of the store paths *paths*, that is,
     the store paths currently existing in the Nix store that refer to
     one of *paths*. Note that contrary to the references, the set of
     referrers is not constant; it can change as store paths are added or
     removed.
 
-  - `--referrers-closure`  
+  - `--referrers-closure`\
     Prints the closure of the set of store paths *paths* under the
     referrers relation; that is, all store paths that directly or
     indirectly refer to one of *paths*. These are all the path currently
     in the Nix store that are dependent on *paths*.
 
-  - `--deriver`; `-d`  
+  - `--deriver`; `-d`\
     Prints the [deriver](../glossary.md) of the store paths *paths*. If
     the path has no deriver (e.g., if it is a source file), or if the
     deriver is not known (e.g., in the case of a binary-only
     deployment), the string `unknown-deriver` is printed.
 
-  - `--graph`  
+  - `--graph`\
     Prints the references graph of the store paths *paths* in the format
     of the `dot` tool of AT\&T's [Graphviz
     package](http://www.graphviz.org/). This can be used to visualise
@@ -364,39 +364,39 @@ symlink.
     this to a store derivation. To obtain a runtime dependency graph,
     apply it to an output path.
 
-  - `--tree`  
+  - `--tree`\
     Prints the references graph of the store paths *paths* as a nested
     ASCII tree. References are ordered by descending closure size; this
     tends to flatten the tree, making it more readable. The query only
     recurses into a store path when it is first encountered; this
     prevents a blowup of the tree representation of the graph.
 
-  - `--graphml`  
+  - `--graphml`\
     Prints the references graph of the store paths *paths* in the
     [GraphML](http://graphml.graphdrawing.org/) file format. This can be
     used to visualise dependency graphs. To obtain a build-time
     dependency graph, apply this to a store derivation. To obtain a
     runtime dependency graph, apply it to an output path.
 
-  - `--binding` *name*; `-b` *name*  
+  - `--binding` *name*; `-b` *name*\
     Prints the value of the attribute *name* (i.e., environment
     variable) of the store derivations *paths*. It is an error for a
     derivation to not have the specified attribute.
 
-  - `--hash`  
+  - `--hash`\
     Prints the SHA-256 hash of the contents of the store paths *paths*
     (that is, the hash of the output of `nix-store --dump` on the given
     paths). Since the hash is stored in the Nix database, this is a fast
     operation.
 
-  - `--size`  
+  - `--size`\
     Prints the size in bytes of the contents of the store paths *paths*
     — to be precise, the size of the output of `nix-store --dump` on
     the given paths. Note that the actual disk space required by the
     store paths may be higher, especially on filesystems with large
     cluster sizes.
 
-  - `--roots`  
+  - `--roots`\
     Prints the garbage collector roots that point, directly or
     indirectly, at the store paths *paths*.
 
@@ -513,7 +513,7 @@ public url or broke since the download expression was written.
 
 This operation has the following options:
 
-  - `--recursive`  
+  - `--recursive`\
     Use recursive instead of flat hashing mode, used when adding
     directories to the store.
 
@@ -540,14 +540,14 @@ being modified by non-Nix tools, or of bugs in Nix itself.
 
 This operation has the following options:
 
-  - `--check-contents`  
+  - `--check-contents`\
     Checks that the contents of every valid store path has not been
     altered by computing a SHA-256 hash of the contents and comparing it
     with the hash stored in the Nix database at build time. Paths that
     have been modified are printed out. For large stores,
     `--check-contents` is obviously quite slow.
 
-  - `--repair`  
+  - `--repair`\
     If any valid path is missing from the store, or (if
     `--check-contents` is given) the contents of a valid path has been
     modified, then try to repair the path by redownloading it. See
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index bc8eb6796..47862bc09 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -2,56 +2,56 @@
 
 Most Nix commands accept the following command-line options:
 
-  - `--help`  
+  - `--help`\
     Prints out a summary of the command syntax and exits.
 
-  - `--version`  
+  - `--version`\
     Prints out the Nix version number on standard output and exits.
 
-  - `--verbose` / `-v`  
+  - `--verbose` / `-v`\
     Increases the level of verbosity of diagnostic messages printed on
     standard error. For each Nix operation, the information printed on
     standard output is well-defined; any diagnostic information is
     printed on standard error, never on standard output.
-    
+
     This option may be specified repeatedly. Currently, the following
     verbosity levels exist:
-    
-      - 0  
+
+      - 0\
         “Errors only”: only print messages explaining why the Nix
         invocation failed.
-    
-      - 1  
+
+      - 1\
         “Informational”: print *useful* messages about what Nix is
         doing. This is the default.
-    
-      - 2  
+
+      - 2\
         “Talkative”: print more informational messages.
-    
-      - 3  
+
+      - 3\
         “Chatty”: print even more informational messages.
-    
-      - 4  
+
+      - 4\
         “Debug”: print debug information.
-    
-      - 5  
+
+      - 5\
         “Vomit”: print vast amounts of debug information.
 
-  - `--quiet`  
+  - `--quiet`\
     Decreases the level of verbosity of diagnostic messages printed on
     standard error. This is the inverse option to `-v` / `--verbose`.
-    
+
     This option may be specified repeatedly. See the previous verbosity
     levels list.
 
-  - `--log-format` *format*  
+  - `--log-format` *format*\
     This option can be used to change the output of the log format, with
     *format* being one of:
-    
-      - raw  
+
+      - raw\
         This is the raw format, as outputted by nix-build.
-    
-      - internal-json  
+
+      - internal-json\
         Outputs the logs in a structured manner.
 
         > **Warning**
@@ -60,30 +60,30 @@ Most Nix commands accept the following command-line options:
         > the error-messages (namely of the `msg`-field) can change
         > between releases.
 
-      - bar  
+      - bar\
         Only display a progress bar during the builds.
-    
-      - bar-with-logs  
+
+      - bar-with-logs\
         Display the raw logs, with the progress bar at the bottom.
 
-  - `--no-build-output` / `-Q`  
+  - `--no-build-output` / `-Q`\
     By default, output written by builders to standard output and
     standard error is echoed to the Nix command's standard error. This
     option suppresses this behaviour. Note that the builder's standard
     output and error are always written to a log file in
     `prefix/nix/var/log/nix`.
 
-  - `--max-jobs` / `-j` *number*  
+  - `--max-jobs` / `-j` *number*\
     Sets the maximum number of build jobs that Nix will perform in
     parallel to the specified number. Specify `auto` to use the number
     of CPUs in the system. The default is specified by the `max-jobs`
     configuration setting, which itself defaults to `1`. A higher
     value is useful on SMP systems or to exploit I/O latency.
-    
+
     Setting it to `0` disallows building on the local machine, which is
     useful when you want builds to happen only on remote builders.
 
-  - `--cores`  
+  - `--cores`\
     Sets the value of the `NIX_BUILD_CORES` environment variable in
     the invocation of builders. Builders can use this variable at
     their discretion to control the maximum amount of parallelism. For
@@ -94,18 +94,18 @@ Most Nix commands accept the following command-line options:
     means that the builder should use all available CPU cores in the
     system.
 
-  - `--max-silent-time`  
+  - `--max-silent-time`\
     Sets the maximum number of seconds that a builder can go without
     producing any data on standard output or standard error. The
     default is specified by the `max-silent-time` configuration
     setting. `0` means no time-out.
 
-  - `--timeout`  
+  - `--timeout`\
     Sets the maximum number of seconds that a builder can run. The
     default is specified by the `timeout` configuration setting. `0`
     means no timeout.
 
-  - `--keep-going` / `-k`  
+  - `--keep-going` / `-k`\
     Keep going in case of failed builds, to the greatest extent
     possible. That is, if building an input of some derivation fails,
     Nix will still build the other inputs, but not the derivation
@@ -113,17 +113,17 @@ Most Nix commands accept the following command-line options:
     for builds of substitutes), possibly killing builds in progress (in
     case of parallel or distributed builds).
 
-  - `--keep-failed` / `-K`  
+  - `--keep-failed` / `-K`\
     Specifies that in case of a build failure, the temporary directory
     (usually in `/tmp`) in which the build takes place should not be
     deleted. The path of the build directory is printed as an
     informational message.
 
-  - `--fallback`  
+  - `--fallback`\
     Whenever Nix attempts to build a derivation for which substitutes
     are known for each output path, but realising the output paths
     through the substitutes fails, fall back on building the derivation.
-    
+
     The most common scenario in which this is useful is when we have
     registered substitutes in order to perform binary distribution from,
     say, a network repository. If the repository is down, the
@@ -134,12 +134,12 @@ Most Nix commands accept the following command-line options:
     failure in obtaining the substitutes to lead to a full build from
     source (with the related consumption of resources).
 
-  - `--readonly-mode`  
+  - `--readonly-mode`\
     When this option is used, no attempt is made to open the Nix
     database. Most Nix operations do need database access, so those
     operations will fail.
 
-  - `--arg` *name* *value*  
+  - `--arg` *name* *value*\
     This option is accepted by `nix-env`, `nix-instantiate`,
     `nix-shell` and `nix-build`. When evaluating Nix expressions, the
     expression evaluator will automatically try to call functions that
@@ -151,7 +151,7 @@ Most Nix commands accept the following command-line options:
     override a default value). That is, if the evaluator encounters a
     function with an argument named *name*, it will call it with value
     *value*.
-    
+
     For instance, the top-level `default.nix` in Nixpkgs is actually a
     function:
 
@@ -161,7 +161,7 @@ Most Nix commands accept the following command-line options:
       ...
     }: ...
     ```
-    
+
     So if you call this Nix expression (e.g., when you do `nix-env -i
     pkgname`), the function will be called automatically using the
     value [`builtins.currentSystem`](../expressions/builtins.md) for
@@ -170,13 +170,13 @@ Most Nix commands accept the following command-line options:
     since the argument is a Nix string literal, you have to escape the
     quotes.)
 
-  - `--argstr` *name* *value*  
+  - `--argstr` *name* *value*\
     This option is like `--arg`, only the value is not a Nix
     expression but a string. So instead of `--arg system
     \"i686-linux\"` (the outer quotes are to keep the shell happy) you
     can say `--argstr system i686-linux`.
 
-  - `--attr` / `-A` *attrPath*  
+  - `--attr` / `-A` *attrPath*\
     Select an attribute from the top-level Nix expression being
     evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and
     `nix-shell` only.) The *attribute path* *attrPath* is a sequence
@@ -185,34 +185,34 @@ Most Nix commands accept the following command-line options:
     would cause the expression `e.xorg.xorgserver` to be used. See
     [`nix-env --install`](nix-env.md#operation---install) for some
     concrete examples.
-    
+
     In addition to attribute names, you can also specify array indices.
     For instance, the attribute path `foo.3.bar` selects the `bar`
     attribute of the fourth element of the array in the `foo` attribute
     of the top-level expression.
 
-  - `--expr` / `-E`  
+  - `--expr` / `-E`\
     Interpret the command line arguments as a list of Nix expressions to
     be parsed and evaluated, rather than as a list of file names of Nix
     expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.)
-    
+
     For `nix-shell`, this option is commonly used to give you a shell in
     which you can build the packages returned by the expression. If you
     want to get a shell which contain the *built* packages ready for
     use, give your expression to the `nix-shell -p` convenience flag
     instead.
 
-  - `-I` *path*  
+  - `-I` *path*\
     Add a path to the Nix expression search path. This option may be
     given multiple times. See the `NIX_PATH` environment variable for
     information on the semantics of the Nix search path. Paths added
     through `-I` take precedence over `NIX_PATH`.
 
-  - `--option` *name* *value*  
+  - `--option` *name* *value*\
     Set the Nix configuration option *name* to *value*. This overrides
     settings in the Nix configuration file (see nix.conf5).
 
-  - `--repair`  
+  - `--repair`\
     Fix corrupted or missing store paths by redownloading or rebuilding
     them. Note that this is slow because it requires computing a
     cryptographic hash of the contents of every path in the closure of
diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md
index 31ebadda1..8a30c42a0 100644
--- a/doc/manual/src/expressions/advanced-attributes.md
+++ b/doc/manual/src/expressions/advanced-attributes.md
@@ -2,14 +2,14 @@
 
 Derivations can declare some infrequently used optional attributes.
 
-  - `allowedReferences`  
+  - `allowedReferences`\
     The optional attribute `allowedReferences` specifies a list of legal
     references (dependencies) of the output of the builder. For example,
-    
+
     ```nix
     allowedReferences = [];
     ```
-    
+
     enforces that the output of a derivation cannot have any runtime
     dependencies on its inputs. To allow an output to have a runtime
     dependency on itself, use `"out"` as a list item. This is used in
@@ -17,45 +17,45 @@ Derivations can declare some infrequently used optional attributes.
     booting Linux don’t have accidental dependencies on other paths in
     the Nix store.
 
-  - `allowedRequisites`  
+  - `allowedRequisites`\
     This attribute is similar to `allowedReferences`, but it specifies
     the legal requisites of the whole closure, so all the dependencies
     recursively. For example,
-    
+
     ```nix
     allowedRequisites = [ foobar ];
     ```
-    
+
     enforces that the output of a derivation cannot have any other
     runtime dependency than `foobar`, and in addition it enforces that
     `foobar` itself doesn't introduce any other dependency itself.
 
-  - `disallowedReferences`  
+  - `disallowedReferences`\
     The optional attribute `disallowedReferences` specifies a list of
     illegal references (dependencies) of the output of the builder. For
     example,
-    
+
     ```nix
     disallowedReferences = [ foo ];
     ```
-    
+
     enforces that the output of a derivation cannot have a direct
     runtime dependencies on the derivation `foo`.
 
-  - `disallowedRequisites`  
+  - `disallowedRequisites`\
     This attribute is similar to `disallowedReferences`, but it
     specifies illegal requisites for the whole closure, so all the
     dependencies recursively. For example,
-    
+
     ```nix
     disallowedRequisites = [ foobar ];
     ```
-    
+
     enforces that the output of a derivation cannot have any runtime
     dependency on `foobar` or any other derivation depending recursively
     on `foobar`.
 
-  - `exportReferencesGraph`  
+  - `exportReferencesGraph`\
     This attribute allows builders access to the references graph of
     their inputs. The attribute is a list of inputs in the Nix store
     whose references graph the builder needs to know. The value of
@@ -65,17 +65,17 @@ Derivations can declare some infrequently used optional attributes.
     files have the format used by `nix-store --register-validity`
     (with the deriver fields left empty). For example, when the
     following derivation is built:
-    
+
     ```nix
     derivation {
       ...
       exportReferencesGraph = [ "libfoo-graph" libfoo ];
     };
     ```
-    
+
     the references graph of `libfoo` is placed in the file
     `libfoo-graph` in the temporary build directory.
-    
+
     `exportReferencesGraph` is useful for builders that want to do
     something with the closure of a store path. Examples include the
     builders in NixOS that generate the initial ramdisk for booting
@@ -84,66 +84,66 @@ Derivations can declare some infrequently used optional attributes.
     with a Nix store containing the closure of a bootable NixOS
     configuration).
 
-  - `impureEnvVars`  
+  - `impureEnvVars`\
     This attribute allows you to specify a list of environment variables
     that should be passed from the environment of the calling user to
     the builder. Usually, the environment is cleared completely when the
     builder is executed, but with this attribute you can allow specific
     environment variables to be passed unmodified. For example,
     `fetchurl` in Nixpkgs has the line
-    
+
     ```nix
     impureEnvVars = [ "http_proxy" "https_proxy" ... ];
     ```
-    
+
     to make it use the proxy server configuration specified by the user
     in the environment variables `http_proxy` and friends.
-    
+
     This attribute is only allowed in *fixed-output derivations* (see
     below), where impurities such as these are okay since (the hash
     of) the output is known in advance. It is ignored for all other
     derivations.
-    
+
     > **Warning**
-    > 
+    >
     > `impureEnvVars` implementation takes environment variables from
     > the current builder process. When a daemon is building its
     > environmental variables are used. Without the daemon, the
     > environmental variables come from the environment of the
     > `nix-build`.
 
-  - `outputHash`; `outputHashAlgo`; `outputHashMode`  
+  - `outputHash`; `outputHashAlgo`; `outputHashMode`\
     These attributes declare that the derivation is a so-called
     *fixed-output derivation*, which means that a cryptographic hash of
     the output is already known in advance. When the build of a
     fixed-output derivation finishes, Nix computes the cryptographic
     hash of the output and compares it to the hash declared with these
     attributes. If there is a mismatch, the build fails.
-    
+
     The rationale for fixed-output derivations is derivations such as
     those produced by the `fetchurl` function. This function downloads a
     file from a given URL. To ensure that the downloaded file has not
     been modified, the caller must also specify a cryptographic hash of
     the file. For example,
-    
+
     ```nix
     fetchurl {
       url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
       sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
     }
     ```
-    
+
     It sometimes happens that the URL of the file changes, e.g., because
     servers are reorganised or no longer available. We then must update
     the call to `fetchurl`, e.g.,
-    
+
     ```nix
     fetchurl {
       url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
       sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
     }
     ```
-    
+
     If a `fetchurl` derivation was treated like a normal derivation, the
     output paths of the derivation and *all derivations depending on it*
     would change. For instance, if we were to change the URL of the
@@ -151,16 +151,16 @@ Derivations can declare some infrequently used optional attributes.
     other packages depend) massive rebuilds would be needed. This is
     unfortunate for a change which we know cannot have a real effect as
     it propagates upwards through the dependency graph.
-    
+
     For fixed-output derivations, on the other hand, the name of the
     output path only depends on the `outputHash*` and `name` attributes,
     while all other attributes are ignored for the purpose of computing
     the output path. (The `name` attribute is included because it is
     part of the path.)
-    
+
     As an example, here is the (simplified) Nix expression for
     `fetchurl`:
-    
+
     ```nix
     { stdenv, curl }: # The curl program is used for downloading.
 
@@ -180,43 +180,43 @@ Derivations can declare some infrequently used optional attributes.
       inherit url;
     }
     ```
-    
+
     The `outputHashAlgo` attribute specifies the hash algorithm used to
     compute the hash. It can currently be `"sha1"`, `"sha256"` or
     `"sha512"`.
-    
+
     The `outputHashMode` attribute determines how the hash is computed.
     It must be one of the following two values:
-    
-      - `"flat"`  
+
+      - `"flat"`\
         The output must be a non-executable regular file. If it isn’t,
         the build fails. The hash is simply computed over the contents
         of that file (so it’s equal to what Unix commands like
         `sha256sum` or `sha1sum` produce).
-        
+
         This is the default.
-    
-      - `"recursive"`  
+
+      - `"recursive"`\
         The hash is computed over the NAR archive dump of the output
         (i.e., the result of [`nix-store
         --dump`](../command-ref/nix-store.md#operation---dump)). In
         this case, the output can be anything, including a directory
         tree.
-    
+
     The `outputHash` attribute, finally, must be a string containing
     the hash in either hexadecimal or base-32 notation. (See the
     [`nix-hash` command](../command-ref/nix-hash.md) for information
     about converting to and from base-32 notation.)
 
-  - `passAsFile`  
+  - `passAsFile`\
     A list of names of attributes that should be passed via files rather
     than environment variables. For example, if you have
-    
+
     ```nix
     passAsFile = ["big"];
     big = "a very long string";
     ```
-    
+
     then when the builder runs, the environment variable `bigPath`
     will contain the absolute path to a temporary file containing `a
     very long string`. That is, for any attribute *x* listed in
@@ -226,7 +226,7 @@ Derivations can declare some infrequently used optional attributes.
     builder, since most operating systems impose a limit on the size
     of the environment (typically, a few hundred kilobyte).
 
-  - `preferLocalBuild`  
+  - `preferLocalBuild`\
     If this attribute is set to `true` and [distributed building is
     enabled](../advanced-topics/distributed-builds.md), then, if
     possible, the derivaton will be built locally instead of forwarded
@@ -234,14 +234,14 @@ Derivations can declare some infrequently used optional attributes.
     where the cost of doing a download or remote build would exceed
     the cost of building locally.
 
-  - `allowSubstitutes`  
+  - `allowSubstitutes`\
     If this attribute is set to `false`, then Nix will always build this
     derivation; it will not try to substitute its outputs. This is
     useful for very trivial derivations (such as `writeText` in Nixpkgs)
     that are cheaper to build than to substitute from a binary cache.
-    
+
     > **Note**
-    > 
+    >
     > You need to have a builder configured which satisfies the
     > derivation’s `system` attribute, since the derivation cannot be
     > substituted. Thus it is usually a good idea to align `system` with
diff --git a/doc/manual/src/expressions/builtin-constants.md b/doc/manual/src/expressions/builtin-constants.md
index 3345a715b..1404289e5 100644
--- a/doc/manual/src/expressions/builtin-constants.md
+++ b/doc/manual/src/expressions/builtin-constants.md
@@ -2,7 +2,7 @@
 
 Here are the constants built into the Nix expression evaluator:
 
-  - `builtins`  
+  - `builtins`\
     The set `builtins` contains all the built-in functions and values.
     You can use `builtins` to test for the availability of features in
     the Nix installation, e.g.,
@@ -14,7 +14,7 @@ Here are the constants built into the Nix expression evaluator:
     This allows a Nix expression to fall back gracefully on older Nix
     installations that don’t have the desired built-in function.
 
-  - `builtins.currentSystem`  
+  - `builtins.currentSystem`\
     The built-in value `currentSystem` evaluates to the Nix platform
     identifier for the Nix installation on which the expression is being
     evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md
index 0f7c3d32f..c16b2805f 100644
--- a/doc/manual/src/expressions/builtins-prefix.md
+++ b/doc/manual/src/expressions/builtins-prefix.md
@@ -9,7 +9,7 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `derivation` *attrs*; `builtins.derivation` *attrs*  
+  - `derivation` *attrs*; `builtins.derivation` *attrs*\
 
     `derivation` is described in [its own section](derivations.md).
 
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 56af9cd85..bb350d9de 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -1,48 +1,48 @@
 # Glossary
 
-  - derivation  
+  - derivation\
     A description of a build action. The result of a derivation is a
     store object. Derivations are typically specified in Nix expressions
     using the [`derivation` primitive](expressions/derivations.md). These are
     translated into low-level *store derivations* (implicitly by
     `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
-  - store  
+  - store\
     The location in the file system where store objects live. Typically
     `/nix/store`.
 
-  - store path  
+  - store path\
     The location in the file system of a store object, i.e., an
     immediate child of the Nix store directory.
 
-  - store object  
+  - store object\
     A file that is an immediate child of the Nix store directory. These
     can be regular files, but also entire directory trees. Store objects
     can be sources (objects copied from outside of the store),
     derivation outputs (objects produced by running a build action), or
     derivations (files describing a build action).
 
-  - substitute  
+  - substitute\
     A substitute is a command invocation stored in the Nix database that
     describes how to build a store object, bypassing the normal build
     mechanism (i.e., derivations). Typically, the substitute builds the
     store object by downloading a pre-built version of the store object
     from some server.
 
-  - purity  
+  - purity\
     The assumption that equal Nix derivations when run always produce
     the same output. This cannot be guaranteed in general (e.g., a
     builder can rely on external inputs such as the network or the
     system time) but the Nix model assumes it.
 
-  - Nix expression  
+  - Nix expression\
     A high-level description of software packages and compositions
     thereof. Deploying software using Nix entails writing Nix
     expressions for your packages. Nix expressions are translated to
     derivations that are stored in the Nix store. These derivations can
     then be built.
 
-  - reference  
+  - reference\
     A store path `P` is said to have a reference to a store path `Q` if
     the store object at `P` contains the path `Q` somewhere. The
     *references* of a store path are the set of store paths to which it
@@ -52,11 +52,11 @@
     output paths), whereas an output path only references other output
     paths.
 
-  - reachable  
+  - reachable\
     A store path `Q` is reachable from another store path `P` if `Q`
     is in the *closure* of the *references* relation.
 
-  - closure  
+  - closure\
     The closure of a store path is the set of store paths that are
     directly or indirectly “reachable” from that store path; that is,
     it’s the closure of the path under the *references* relation. For
@@ -71,29 +71,29 @@
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
     references `R` then `R` is also in the closure of `P`.
 
-  - output path  
+  - output path\
     A store path produced by a derivation.
 
-  - deriver  
+  - deriver\
     The deriver of an *output path* is the store
     derivation that built it.
 
-  - validity  
+  - validity\
     A store path is considered *valid* if it exists in the file system,
     is listed in the Nix database as being valid, and if all paths in
     its closure are also valid.
 
-  - user environment  
+  - user environment\
     An automatically generated store object that consists of a set of
     symlinks to “active” applications, i.e., other store paths. These
     are generated automatically by
     [`nix-env`](command-ref/nix-env.md). See *profiles*.
 
-  - profile  
+  - profile\
     A symlink to the current *user environment* of a user, e.g.,
     `/nix/var/nix/profiles/default`.
 
-  - NAR  
+  - NAR\
     A *N*ix *AR*chive. This is a serialisation of a path in the Nix
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`
diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md
index a4f4d561f..30f2b2e11 100644
--- a/doc/manual/src/package-management/s3-substituter.md
+++ b/doc/manual/src/package-management/s3-substituter.md
@@ -7,17 +7,17 @@ cache mechanism that Nix usually uses to fetch prebuilt binaries from
 
 The following options can be specified as URL parameters to the S3 URL:
 
-  - `profile`  
+  - `profile`\
     The name of the AWS configuration profile to use. By default Nix
     will use the `default` profile.
 
-  - `region`  
+  - `region`\
     The region of the S3 bucket. `us–east-1` by default.
     
     If your bucket is not in `us–east-1`, you should always explicitly
     specify the region parameter.
 
-  - `endpoint`  
+  - `endpoint`\
     The URL to your S3-compatible service, for when not using Amazon S3.
     Do not specify this value if you're using Amazon S3.
     
@@ -26,7 +26,7 @@ The following options can be specified as URL parameters to the S3 URL:
     > This endpoint must support HTTPS and will use path-based
     > addressing instead of virtual host based addressing.
 
-  - `scheme`  
+  - `scheme`\
     The scheme used for S3 requests, `https` (default) or `http`. This
     option allows you to disable HTTPS for binary caches which don't
     support it.
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ff2f302ed..647b66ef6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1962,26 +1962,26 @@ static RegisterPrimOp primop_path({
       An enrichment of the built-in path type, based on the attributes
       present in *args*. All are optional except `path`:
 
-        - path  
+        - path\
           The underlying path.
 
-        - name  
+        - name\
           The name of the path when added to the store. This can used to
           reference paths that have nix-illegal characters in their names,
           like `@`.
 
-        - filter  
+        - filter\
           A function of the type expected by `builtins.filterSource`,
           with the same semantics.
 
-        - recursive  
+        - recursive\
           When `false`, when `path` is added to the store it is with a
           flat hash, rather than a hash of the NAR serialization of the
           file. Thus, `path` must refer to a regular file, not a
           directory. This allows similar behavior to `fetchurl`. Defaults
           to `true`.
 
-        - sha256  
+        - sha256\
           When provided, this is the expected hash of the file at the
           path. Evaluation will fail if the hash is incorrect, and
           providing a hash allows `builtins.path` to be used even when the
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 27d8ddf35..b8b99d4fa 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -303,17 +303,17 @@ static RegisterPrimOp primop_fetchGit({
       of the repo at that URL is fetched. Otherwise, it can be an
       attribute with the following attributes (all except `url` optional):
 
-        - url  
+        - url\
           The URL of the repo.
 
-        - name  
+        - name\
           The name of the directory the repo should be exported to in the
           store. Defaults to the basename of the URL.
 
-        - rev  
+        - rev\
           The git revision to fetch. Defaults to the tip of `ref`.
 
-        - ref  
+        - ref\
           The git ref to look for the requested revision under. This is
           often a branch or tag name. Defaults to `HEAD`.
 
@@ -321,11 +321,11 @@ static RegisterPrimOp primop_fetchGit({
           of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
           with `refs/`.
 
-        - submodules  
+        - submodules\
           A Boolean parameter that specifies whether submodules should be
           checked out. Defaults to `false`.
 
-        - allRefs  
+        - allRefs\
           Whether to fetch all refs of the repository. With this argument being
           true, it's possible to load a `rev` from *any* `ref` (by default only
           `rev`s from the specified `ref` are supported).
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3e4ead76c..6f8749254 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -701,7 +701,7 @@ public:
           send a series of commands to modify various settings to stdout. The
           currently recognized commands are:
 
-            - `extra-sandbox-paths`  
+            - `extra-sandbox-paths`\
               Pass a list of files and directories to be included in the
               sandbox for this build. One entry per line, terminated by an
               empty line. Entries have the same format as `sandbox-paths`.

From e6a360a4d92eb428eeb098e7bc970d141de1618a Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 27 Apr 2021 17:24:38 +0200
Subject: [PATCH 882/882] primops/storePath: add trace to pure mode error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As described in #4745 it's otherwise fairly hard to understand where
this is coming from. Say you have an expression which uses e.g.
`types.package`:

``` nix
{ outputs = { self, nixpkgs }: {
    packages.x86_64-linux.hello = let
      foo = nixpkgs.lib.evalModules {
        modules = [
          {
            options.foo.bar = with nixpkgs.lib; mkOption { type = types.package; };
          }
          {
            foo.bar = ./.;
          }
        ];
      };
    in builtins.trace foo.config.foo.bar.outPath nixpkgs.legacyPackages.x86_64-linux.hello;
    defaultPackage.x86_64-linux = self.packages.x86_64-linux.hello;
  };
}
```

Then you'll get an error trace like this:

```
error: 'builtins.storePath' is not allowed in pure evaluation mode

       at /nix/store/p4h2x6r80njkb0j2rc1xjhhl99yri3zb-source/lib/attrsets.nix:328:15:

          327|     let
          328|       path' = builtins.storePath path;
             |               ^
          329|       res =

       … while evaluating the attribute 'config.foo.bar.outPath'

       at /nix/store/p4h2x6r80njkb0j2rc1xjhhl99yri3zb-source/lib/attrsets.nix:332:11:

          331|           name = sanitizeDerivationName (builtins.substring 33 (-1) (baseNameOf path'));
          332|           outPath = path';
             |           ^
          333|           outputs = [ "out" ];

       … while evaluating the attribute 'packages.x86_64-linux.hello'

       at /nix/store/6c1rfsqzrhjw1235palzjmf5vihcpci7-source/flake.nix:3:5:

            2| { outputs = { self, nixpkgs }: {
            3|     packages.x86_64-linux.hello = let
             |     ^
            4|       foo = nixpkgs.lib.evalModules {
```

Fixes #4745
---
 src/libexpr/primops.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 428adf4c2..1b9948eaf 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1210,7 +1210,10 @@ static RegisterPrimOp primop_toPath({
 static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     if (evalSettings.pureEval)
-        throw EvalError("builtins.storePath' is not allowed in pure evaluation mode");
+        throw EvalError({
+            .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"),
+            .errPos = pos
+        });
 
     PathSet context;
     Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context));